diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index beac79f..f201925 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: with: ty-src: omop_alchemy build-test-postgres: - uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-postgres.yml@main + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-postgres-v2.yml@main with: ty-src: omop_alchemy postgres-db: omop_alchemy_ci @@ -43,14 +43,11 @@ jobs: --password test \ --database-name omop_alchemy_test_ci \ --test-only true - uv run omop-config databases add cdm_db \ - --kind cdm \ + uv run omop-config databases add cdm cdm_db \ --connection ci_pg \ - --schema-name public - uv run omop-config databases add test_cdm_db \ - --kind cdm \ + --cdm-schema public + uv run omop-config databases add cdm test_cdm_db \ --connection ci_pg_test \ - --schema-name public + --cdm-schema public uv run omop-config configure omop_alchemy \ - --cdm-db cdm_db \ - --test-cdm-db test_cdm_db + --test-cdm-db-pg test_cdm_db diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d20e252..0000000 --- a/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM python:3.12-slim -RUN pip install --no-cache-dir ".[postgres]" -WORKDIR /workspace diff --git a/docs/advanced/backends.md b/docs/advanced/backends.md index e69de29..a42e9c0 100644 --- a/docs/advanced/backends.md +++ b/docs/advanced/backends.md @@ -0,0 +1,41 @@ +# Backend Compatibility + +Every maintenance operation goes through a `Backend` (`PostgresBackend` or +`SQLiteBackend`). PostgreSQL implements the full `Backend` interface. +SQLite implements only what SQLite itself can do; everything else raises +`FeatureNotSupportedError` at call time rather than failing silently or +producing a partial result. + +| Feature | PostgreSQL | SQLite | +| --- | --- | --- | +| Index existence check | Yes | Yes | +| Drop index if exists | Yes | Yes (unqualified — no schema concept) | +| `ANALYZE` | Yes | Yes | +| `VACUUM ANALYZE` | Yes | No | +| FK trigger management / status | Yes | No | +| FK constraint violation counting | Yes | No | +| Table clustering (`CLUSTER`) | Yes | No | +| Cluster index inspection | Yes | No | +| Functional-index expression normalization | Yes | No (SQLite never reflects expression-based indexes) | +| `TRUNCATE ... RESTART IDENTITY / CASCADE` | Yes | No | +| Sequence lookup / reset | Yes | No (SQLite has no sequences) | +| Full-text search | Yes — see [PostgreSQL Full-Text Search](fulltext.md) | No | +| Database backup / restore | Yes | No | + +## What this means in practice + +Maintenance CLI commands that rely on a not-supported feature raise +`FeatureNotSupportedError` on SQLite rather than doing nothing. In +particular, against a SQLite database: + +- `indexes cluster` and the clustering step of `manage_indexes --enable` + are unavailable. +- `truncate-tables` cannot use `RESTART IDENTITY`/`CASCADE`. +- `fulltext install` is unavailable entirely. +- `backup-database`/`restore-database` are unavailable entirely. +- FK trigger toggling and FK violation counting are unavailable. + +SQLite remains fully supported for the core ORM/CDM layer (models, queries, +`create_missing_tables`, schema-provenance guarding) — these limitations are +specific to the maintenance operations listed above, which assume a +PostgreSQL-grade catalog. diff --git a/docs/advanced/index.md b/docs/advanced/index.md index c2f08d5..976c809 100644 --- a/docs/advanced/index.md +++ b/docs/advanced/index.md @@ -17,3 +17,4 @@ the immutability and interpretability of the underlying CDM tables. - [Backend Compatibility](backends.md) - [PostgreSQL Full-Text Search](fulltext.md) +- [Vocabulary Load Performance](vocabulary_load_performance.md) diff --git a/docs/cli/index.md b/docs/cli/index.md index 5548371..f64bdb0 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -90,7 +90,7 @@ With the decorator, the function body is all that matters: @app.command("my-command") @omop_command("my-command") def my_command(conn, engine) -> None: - results = do_work(engine, db_schema=conn.db_schema) + results = do_work(engine, db_schema=conn.resolved.schema_name) console.print(render_results(results)) ``` @@ -102,5 +102,5 @@ def my_command(conn, engine) -> None: | Attribute | Description | |---|---| -| `conn.db_schema` | CDM schema name from the resolved database (e.g. `"omop"`) | +| `conn.resolved` | The resolved `ResolvedCDMDatabase`; `conn.resolved.schema_name` is the CDM schema name (e.g. `"omop"`) | | `conn.athena_source` | Athena vocabulary CSV directory from `[tools.omop_alchemy]`'s `athena_source_path` field; `None` if not configured | diff --git a/docs/getting-started/common-use-cases.md b/docs/getting-started/common-use-cases.md new file mode 100644 index 0000000..a729a7f --- /dev/null +++ b/docs/getting-started/common-use-cases.md @@ -0,0 +1,130 @@ +# Common Use Cases + +This pages details common use-cases and setups for users and how to wrap their established OMOP CDM with `omop-alchemy` and configure it with `oa-configurator`. + +!!! note "Important References" + - [**`oa-configurator` config reference**](https://AustralianCancerDataNetwork.github.io/oa-configurator/config-reference/): Information about the config file created and stored by default at `~/.config/omop/config.toml` + - [**`omop-alchemy` and relationship to OMOP CDM**](configuration.md#cdm-table-roles): Important how schemas capture specific tables. + - [**`oa_configurator`'s architecture guide`**](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/): Information about the core concepts, supported configuration templates, schema translation and provenance guard, and more. + - [**`omop-alchemy`'s maintenance module**](maintenance.md): full command reference + + +--- + +## Vocabulary tables in a separate schema, same server + +!!! example "Scenario" + - Your `Concept` table (and the rest of the vocabulary) lives in a `myvocab` schema + - Vocabulary is separate from your clinical tables' schema. + - [Reading how `omop-alchemy` bundles tables in schemas](configuration.md#cdm-table-roles) reveals, that the `vocab_schema` configuration key is responsible for the `Concept` table + +### Solution +Set `vocab_schema` to `myvocab` during interactive configuration +```bash +omop-config configure omop_alchemy +``` + +The resulting entry in `config.toml` will look like this: + +```toml +[connections.] +dialect = ... +host = ... +.... + +[databases.] +kind = "cdm" +connection = "" +cdm_schema = "" +vocab_schema = "myvocab" # <- overwritten schema map + +[tools.omop_alchemy] +cdm_db = "" +``` + +Every vocabulary-tagged table (see [Documentation for more details](configuration.md#cdm-table-roles)) now resolves into `myvocab` automatically. +This does not require any model changes. `results_schema` works the same way for results tables and is also defined in the [Documentation](configuration.md#cdm-table-roles) + +### Troubleshooting + +#### 1. You misconfigured the schema wrong for your select database + +No issues. Just re-run the configuration command again: +```bash +omop-config configure omop_alchemy +``` + +The CLI wizard will guide you through the entire setup again. You can changed/modify settings. Previously configured fields are now the default and can just be accepted by pressing 'Enter'. + +--- + +## Vocabulary on an entirely separate server + +!!! example "Scenario" + - Your entire CDM vocabulary lives on a separate physical DB server (e.g. a shared vocabnulary instance resued across multiple CDM deployments) + - You checked the documentation for [supported dialects in `omop-alchemy`](https://AustralianCancerDataNetwork.github.io/oa-configurator/config-reference/#supported-dialects ) and confirmed that your separate DB server is supported + + +### Solution + +Configure a separate `vocab_connection` during the setup of `omop-alchemy` when prompted: +```bash +omop-config configure omop_alchemy +``` + +```toml +[connections.cdm] # <- your CDM connection +dialect = "postgresql+psycopg" +host = "cdm-db.internal" +database_name = "cdm" + +[connections.vocab] # <- your vocab connection +dialect = "postgresql+psycopg" +host = "vocab-db.internal" +database_name = "vocab" + +[databases.cdm_db] +kind = "cdm" +connection = "cdm" +vocab_connection = "vocab" # <- references your vocabulary DB +``` + +!!! warning "Queries spanning both databases" + Reads that only touch vocabulary tables route to the `vocab` connection automatically. However, a query joining a vocabulary table against a clinical table (that lives in the `cdm` connection/DB) can't be answered by one physical connection in the underlying backend `SQLAlchemy`. `omop-alchemy` resolves those by querying each side separately and then merging the results in Python. Large queries therefore require significant memory budgets depending on the query. + +--- + +## Migrating an existing deployment to a schema split + +!!! example "Scenario" + - You are moving from one schema holding everything to a real vocabulary/results splits, **or** + - You are renaming a schema on a database `omop-alchemy` has already created tables in. + - **Assumptions:** + - your database for the CDM is named `my_db` in `config.toml` + - there is an entry called `[databases.my_db]`, and + - `[tools.omop_alchemy]` lists it as `cdm_db="my_db"` + - you want to move all your tables governed by the interal `vocab` schema to schema `myvocab` + +### Solution + +[`oa-configurator`'s schema provenance guard](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/#schema-provenance-guard) records which physical schema each role last resolved to, and refuses to run `create-missing-tables` if the configured schema for a role has silently changed since the last run. This mechanism is in place to stop a misconfiguration from creating an orphaned second copy of your tables. To make a genuine change deliberately: + +1. Update `vocab_schema`/`results_schema`/`cdm_schema` in `config.toml` through reconfiguration + ```bash + omop-config configure omop_alchemy + ``` +2. Move your actual data to the new schema yourself using access to the database. + - This is **NEVER** done automatically to preserve data integrity from our end. +3. Record the new schema as the accepted baseline following the assumptions listed in "Scenario" above: + ```bash + omop-config acknowledge-schema-migration --database my_db --schema-tag vocab --new-schema myvocab --reason "moving vocab off the shared schema" + ``` +4. Once you've confirmed the new schema is correct, clean up the old one: + ```bash + omop-config drop-orphan-schema-tables --database cdm_db --schema old_vocab_schema --confirm + ``` + Omit `--confirm` first to preview what would be dropped. + + Both commands live in `oa-configurator`, not `omop-alchemy` as they're generic over any `[databases.*]` entry, not CDM-specific. + + diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 20241ca..8fc07bf 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -29,9 +29,9 @@ database_name = "omop_cdm" test_only = false [databases.cdm_db] -kind = "cdm" -connection = "cdm" -schema_name = "omop" +kind = "cdm" +connection = "cdm" +cdm_schema = "omop" [tools.omop_alchemy] cdm_db = "cdm_db" @@ -39,6 +39,29 @@ cdm_db = "cdm_db" You can also write or edit this file manually. It follows the `oa-configurator` pattern of [physical]->[logical] resource definition, where one connection may serve multiple databases, and each application may define its own database resource, or choose to cross reference an existing one that will be resolved upon connection in the consuming application. +## CDM table roles + +OMOP_Alchemy tags every table with a logical role, matching the [OMOP CDM v5.4](https://ohdsi.github.io/CommonDataModel/cdm54.html) +categories: + +- **Clinical/derived tables** (`Role.PRIMARY`): + - All other tables not captured by the configurations below + - Controlled by `cdm_schema` in the configuration. +- **Vocabulary tables** (`Role.VOCAB`): + - `concept`, `concept_ancestor`, `concept_class`, `concept_relationship`, `concept_synonym`, `domain`, `drug_strength`, `relationship`, `source_to_concept_map`, `vocabulary` + - Controlled by `vocab_schema` in the configuration. +- **Results/analytics tables** (`Role.RESULTS`): + - `cohort`, `cohort_definition` + - Controlled by `results_schema` in the configuration. + +![OMOP CDM v5.4](https://ohdsi.github.io/CommonDataModel/man/images/cdm55.png) + +Each role folds back to `cdm_schema` when its own field is unset, so a minimal config needs no extra fields. +Setting `vocab_schema`/`results_schema` routes just that role's tables elsewhere. See [oa_configurator's schema translate map guide](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/#schema-translate-map) for how the routing itself works, and [Common Use Cases](common-use-cases.md) for worked examples of splitting these onto different schemas or servers. + +!!! info "Misconfiguration prevention" + Misconfiguring which schema a role points at doesn't corrupt data. [`oa-configurator`'s schema provenance guard](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/#schema-provenance-guard) refuses the DDL. + ## Vocabulary loading If you plan to load OMOP vocabulary from Athena CSV files, add the path to `[tools.omop_alchemy]`: @@ -76,5 +99,6 @@ See the [oa-configurator integration guide](https://AustralianCancerDataNetwork. ## Further reading +- [Common Use Cases](common-use-cases.md): worked examples for vocab/results schema splits, a separate vocabulary server, and migrating an existing deployment's schema layout - [oa_configurator quickstart](https://AustralianCancerDataNetwork.github.io/oa-configurator/quickstart/): full config reference, CLI walkthrough - [oa_configurator integration guide](https://AustralianCancerDataNetwork.github.io/oa-configurator/integration/): multi-package setups diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 39c58f6..c085980 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -9,5 +9,7 @@ These pages cover installation, maintenance tooling, and a minimal quickstart fo ## Orientation - [Installation](installation.md) +- [Configuration](configuration.md) - [Maintenance CLI](maintenance.md) +- [Common Use Cases](common-use-cases.md) - [Quickstart](quickstart.md) diff --git a/docs/getting-started/maintenance.md b/docs/getting-started/maintenance.md index 336b174..0b4ed88 100644 --- a/docs/getting-started/maintenance.md +++ b/docs/getting-started/maintenance.md @@ -16,7 +16,7 @@ Some commands depend on PostgreSQL-specific features and will return an error if | Command group | Requires PostgreSQL | Why | | --- | --- | --- | -| `load-vocab-source` | No (PostgreSQL + SQLite) | Uses ORM CSV loader; `--bulk-mode` and `--db-schema` are PostgreSQL-only | +| `load-vocab-source` | No (PostgreSQL + SQLite) | Uses ORM CSV loader; `--bulk-mode` is PostgreSQL-only | | `indexes` | No (cluster apply is PostgreSQL-only) | Index DDL is standard SQL; `CLUSTER` is PostgreSQL | | `create-missing-tables`, `reconcile-schema`, `data-summary`, `info`, `doctor` | No | Pure SQLAlchemy metadata operations | | `reset-sequences` | Yes | PostgreSQL sequences (`SETVAL`) | @@ -271,6 +271,13 @@ omop-alchemy reset-sequences --vocab # vocabulary tables only --- +!!! note "Schema-drift protection" + `fulltext install`, `indexes enable`/`disable`/`cluster`, `truncate-tables`, and + vocabulary-table creation each guard their DDL against the configured schema having + silently drifted since it was last recorded, and raise `SchemaDriftError` if it has. + See [Schema drift](#schema-drift) below for the remediation path + (`omop-config acknowledge-schema-migration`). + ## Command reference | Command | Purpose | Key options | Backend | @@ -290,6 +297,7 @@ omop-alchemy reset-sequences --vocab # vocabulary tables only | `analyze-tables` | Refresh planner statistics | `--scope`, `--table`, `--vacuum` | PostgreSQL, SQLite (`--vacuum` PostgreSQL-only) | | `indexes disable` | Drop ORM-defined secondary indexes | `--vocab`, `--dry-run` | All | | `indexes enable` | Recreate ORM-defined secondary indexes | `--vocab`, `--dry-run` | All (cluster on PostgreSQL) | +| `indexes cluster` | Physically rewrite tables sorted by their cluster index | `--vocab`, `--dry-run` | PostgreSQL | | `fulltext install` | Add tsvector sidecar columns to vocabulary tables | `--regconfig`, `--no-create-indexes` | PostgreSQL | | `fulltext populate` | Populate sidecar tsvector vectors | `--regconfig` | PostgreSQL | | `fulltext drop` | Remove tsvector sidecar columns and indexes | | PostgreSQL | diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 7f24586..83a8a3b 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -18,14 +18,10 @@ See [Configuration](configuration.md) for the full field reference. ## Running PostgreSQL tests locally -The test suite includes PostgreSQL-specific tests that skip automatically unless a `test_cdm_db` database is configured in `~/.config/omop/config.toml`. Tests are marked with `@pytest.mark.requires_database("test_cdm_db")` and skipped at collection time when the database is absent — no manual filtering required. +The test suite includes PostgreSQL-specific tests that skip automatically unless a `test_cdm_db_pg` database is configured in `~/.config/omop/config.toml`. They're resolved via oa-configurator's `isolated_test_database()`, wrapped in this repo's own `pg_db`/`pg_engine`/`pg_session` fixtures, and marked `@pytest.mark.postgresql` (plus `db_dialect` where a test could corrupt shared ORM metadata if run alongside SQLite in the same process). `addopts = "-m 'not db_dialect'"` excludes those by default, so a plain `pytest` run skips them with no manual filtering required. Run them explicitly with `pytest -m postgresql`. -> **This test database is destructive.** The test suite drops and recreates the entire `public` -> schema on every run. `test_cdm_db` must point to a **dedicated, empty test database**, never -> to a database that contains real data. The test suite enforces this: it fails loudly (not skips) if the -> configured database is not marked `test_only = true` in your config. -> -> Refer to the CI/CD workflows at [cava-devops](https://github.com/AustralianCancerDataNetwork/cava-devops/blob/main/.github/workflows/build-test-postgres.yml) for more details on how integration test runs are typically orchestrated. +!!! warning "This test database is destructive." + `pg_session`-backed tests drop and recreate every non-system schema (not just `public`) both before and after each test. `test_cdm_db_pg` must point to a **dedicated, empty test database**, never to a database that contains real data. The test suite enforces this: it fails loudly if the configured database is not marked `test_only = true` in your config. The suite runs sequentially by design and does not support `pytest-xdist`: it fails loudly under `-n 2` or higher rather than racing another worker's reset. **Step 1 — Register a test database connection:** @@ -33,7 +29,7 @@ The test suite includes PostgreSQL-specific tests that skip automatically unless omop-config configure omop_alchemy ``` -When prompted whether to configure a test database, answer **Y** and supply the connection details for your dedicated test PostgreSQL instance. It will be saved as `test_cdm_db` with `test_only = true`. +When prompted whether to configure a test database, answer **Y** and supply the connection details for your dedicated test PostgreSQL instance. It will be saved as `test_cdm_db_pg` with `test_only = true`. > **Note on permissions**: the test suite disables FK constraint triggers during bulk vocabulary > loads, an operation PostgreSQL restricts to superusers. Ensure the test database user has @@ -45,4 +41,4 @@ When prompted whether to configure a test database, answer **Y** and supply the pytest -v tests/ ``` -PostgreSQL tests auto-skip when `test_cdm_db` is not configured; all other tests run regardless. +PostgreSQL tests are excluded from a plain `pytest` run by default (see above); run `pytest -m postgresql` to include them, or `pytest -v tests/ -m postgresql` for verbose output. They still auto-skip if `test_cdm_db_pg` is not configured. diff --git a/docs/models/clinical/observation.md b/docs/models/clinical/observation.md index 809e8aa..6dcb23d 100644 --- a/docs/models/clinical/observation.md +++ b/docs/models/clinical/observation.md @@ -1,4 +1,4 @@ -# oubservation +# observation > Documentation coming soon. diff --git a/docs/models/index.md b/docs/models/index.md index c6563dd..0ba0194 100644 --- a/docs/models/index.md +++ b/docs/models/index.md @@ -43,3 +43,34 @@ validation, and reuse. - [Drug Era](derived/drug_era.md) - [Dose Era](derived/dose_era.md) - [Cohort & Cohort Definitions](derived/cohort.md) + +--- + +## Health Economic + +- [Cost](health_economic/cost.md) +- [Payer Plan Period](health_economic/payer_plan_period.md) + +--- + +## Metadata + +- [CDM Source](metadata/cdm_source.md) +- [Metadata](metadata/metadata.md) + +--- + +## Structural + +- [Episode](structural/episode.md) +- [Episode Event](structural/episode_event.md) +- [Fact Relationship](structural/fact_relationship.md) + +--- + +## Unstructured + +- [Note](unstructured/note.md) +- [Note NLP](unstructured/note_nlp.md) +- [Image](unstructured/image.md) +- [Image Feature](unstructured/image_feature.md) diff --git a/docs/models/vocabulary/index.md b/docs/models/vocabulary/index.md index bea4cf8..72a8586 100644 --- a/docs/models/vocabulary/index.md +++ b/docs/models/vocabulary/index.md @@ -1,4 +1,4 @@ -# vocaubulary Models +# vocabulary Models This section contains ORM models corresponding to OMOP CDM vocabulary tables. diff --git a/docs/models/vocabulary/vocabulary.md b/docs/models/vocabulary/vocabulary.md index 321f8c8..420c6b9 100644 --- a/docs/models/vocabulary/vocabulary.md +++ b/docs/models/vocabulary/vocabulary.md @@ -1,4 +1,4 @@ -# vocaubulary +# vocabulary > Documentation coming soon. diff --git a/docs/toolkit/analytics.md b/docs/toolkit/analytics.md index d7c0f67..8967cac 100644 --- a/docs/toolkit/analytics.md +++ b/docs/toolkit/analytics.md @@ -82,8 +82,7 @@ The single-value properties return the first modality in this order for which th The oncology package publishes lazy governed concept specifications for T, N, M, and group stage, tumour grade, and metastatic disease. Laterality and tumour size are exposed as governed scalar accessors. These declarations consume -`omop-semantics`; the narrow metastatic-disease descendant group requires -`omop-semantics` 0.6.1. Importing the module does not expand a vocabulary or +`omop-semantics` (the package already requires `>=0.6.2`). Importing the module does not expand a vocabulary or contact a database. Stage selection is a query policy over an already filtered canonical modifier diff --git a/docs/toolkit/materialized-views.md b/docs/toolkit/materialized-views.md index f2c828a..13da8ca 100644 --- a/docs/toolkit/materialized-views.md +++ b/docs/toolkit/materialized-views.md @@ -4,7 +4,7 @@ A materialized view is a persisted read model: an expensive or carefully defined ## The deployment contract -The supported deployment contract is PostgreSQL with unqualified materialized-view names resolving to the `public` schema through the connection's `search_path`. Leave `schema` at its default when calling the lifecycle methods. Qualified non-`public` schemas and schema translation are not part of this package's materialized-view contract. +The supported deployment contract is PostgreSQL with unqualified materialized-view names resolving to the `public` schema through the connection's `search_path`. Leave `schema_tag` at its default when calling the lifecycle methods. Qualified non-`public` schemas and schema translation are not part of this package's materialized-view contract. OMOP Alchemy owns the OMOP models and the query-building vocabulary. [`orm-loader`](https://australiancancerdatanetwork.github.io/orm-loader/tables/mat_view/) owns the generic materialized-view definition and database lifecycle. Keeping that boundary means this package can describe an OMOP read model without growing a second implementation of DDL, refresh, index creation, or dependency ordering. diff --git a/mkdocs.yml b/mkdocs.yml index 821b051..9e96b2c 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -70,6 +70,7 @@ nav: - Installation: getting-started/installation.md - Configuration: getting-started/configuration.md - Quickstart: getting-started/quickstart.md + - Common Use Cases: getting-started/common-use-cases.md - Maintenance CLI: getting-started/maintenance.md - CLI Reference: diff --git a/omop_alchemy/backends/__init__.py b/omop_alchemy/backends/__init__.py index f0f980e..78934f0 100644 --- a/omop_alchemy/backends/__init__.py +++ b/omop_alchemy/backends/__init__.py @@ -12,7 +12,7 @@ ) from .postgres import PostgresBackend from .sqlite import SQLiteBackend -from .resolve import resolve_backend, SupportedDialect +from .resolve import resolve_backend __all__ = [ "Backend", @@ -28,5 +28,4 @@ "PostgresBackend", "SQLiteBackend", "resolve_backend", - "SupportedDialect", ] diff --git a/omop_alchemy/backends/base.py b/omop_alchemy/backends/base.py index 8db8571..6de5c1e 100644 --- a/omop_alchemy/backends/base.py +++ b/omop_alchemy/backends/base.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any import sqlalchemy as sa +from oa_configurator import Role if TYPE_CHECKING: from sqlalchemy.sql import ColumnElement @@ -83,9 +84,9 @@ def toggle_fk_triggers( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, enable: bool, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("FK trigger management", self) @@ -93,7 +94,8 @@ def get_fk_trigger_counts( self, conn: sa.Connection, table_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[int, int]: """Return (disabled_count, enabled_count) for RI triggers on the table.""" raise FeatureNotSupportedError("FK trigger status inspection", self) @@ -105,7 +107,9 @@ def count_fk_violations( referred_table: str, constrained_cols: list[str], referred_cols: list[str], - db_schema: str | None, + *, + source_schema_tag: str = Role.PRIMARY.value, + referred_schema_tag: str = Role.PRIMARY.value, ) -> int: raise FeatureNotSupportedError("FK constraint violation counting", self) @@ -116,7 +120,8 @@ def cluster_table( conn: sa.Connection, table_name: str, index_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("Table clustering", self) @@ -124,10 +129,26 @@ def get_clustered_index_name( self, conn: sa.Connection, table_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> str | None: raise FeatureNotSupportedError("Cluster index inspection", self) + # ── Schema reconciliation ──────────────────────────────────────────────── + + def normalize_index_expression(self, sql_text: str) -> str: + """Canonicalize a reflected functional-index expression for comparison + against the ORM's own compiled expression text. + + A backend's own catalog reflection can introduce dialect-specific + canonicalization noise (implicit casts, identifier case) that the + ORM's compiled text never has. Only called when a backend actually + reflects expression-based indexes back with real expression text to + normalize; a backend that can't (e.g. SQLite never reflects them at + all) never reaches this call. + """ + raise FeatureNotSupportedError("Functional-index expression normalization", self) + # ── Table operations ───────────────────────────────────────────────────── @abstractmethod @@ -135,16 +156,17 @@ def analyze_table( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, vacuum: bool = False, + schema_tag: str = Role.PRIMARY.value, ) -> None: ... def index_exists( self, conn: sa.Connection, index_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> bool: """Return True when the named index currently exists on the database. @@ -157,13 +179,17 @@ def drop_index_if_exists( self, conn: sa.Connection, index_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> None: """Drop an index by name without relying on SQLAlchemy's reflection-based checkfirst. Some backends (e.g. SQLite) can't reflect expression-based indexes, so Index.drop(checkfirst=True) would silently no-op on them. IF EXISTS is - evaluated by the database itself, not by reflection. + evaluated by the database itself, not by reflection. schema_tag is + accepted for interface parity with the schema-aware override + (PostgresBackend); this default implementation is unqualified, + matching SQLite having no schema concept to route through. """ conn.exec_driver_sql(f'DROP INDEX IF EXISTS "{index_name}"') @@ -171,10 +197,10 @@ def truncate_table_batch( self, conn: sa.Connection, table_names: list[str], - db_schema: str | None, *, restart_identities: bool, cascade: bool, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("TRUNCATE with RESTART IDENTITY / CASCADE", self) @@ -185,7 +211,8 @@ def find_sequence_name( conn: sa.Connection, table_name: str, column_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> str | None: raise FeatureNotSupportedError("Owned sequence lookup", self) @@ -197,22 +224,6 @@ def set_sequence_value( ) -> None: raise FeatureNotSupportedError("Sequence value reset", self) - # ── Schema context ─────────────────────────────────────────────────────── - - def configure_schema_context( - self, - conn: sa.Connection, - db_schema: str | None, - ) -> None: - pass # no-op by default; PostgreSQL overrides with SET search_path - - def ensure_schema( - self, - conn: sa.Connection, - schema: str | None, - ) -> None: - pass # no-op by default; backends that support named schemas override this - # ── Full-text search ───────────────────────────────────────────────────── @property @@ -247,9 +258,9 @@ def install_fulltext_on_table( table_name: str, vector_column_name: str, index_name: str, - db_schema: str | None, create_indexes: bool, fastupdate: bool, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("Full-text search", self) @@ -260,8 +271,8 @@ def populate_fulltext_on_table( table_name: str, vector_column_name: str, source_column_name: str, - db_schema: str | None, regconfig: str, + schema_tag: str = Role.PRIMARY.value, ) -> int | None: raise FeatureNotSupportedError("Full-text search", self) @@ -272,8 +283,8 @@ def drop_fulltext_on_table( table_name: str, vector_column_name: str, index_name: str, - db_schema: str | None, drop_indexes: bool, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("Full-text search", self) @@ -284,7 +295,8 @@ def prepare_backup( engine: sa.Engine, output_path: str, backup_format: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[str, list[str], dict[str, str], str]: """Return (tool_path, command, env, database_name). subprocess.run stays in CLI.""" raise FeatureNotSupportedError("Database backup", self) @@ -294,7 +306,8 @@ def prepare_restore( engine: sa.Engine, input_path: str, backup_format: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[str, list[str], dict[str, str], str]: """Return (tool_path, command, env, database_name). subprocess.run stays in CLI.""" raise FeatureNotSupportedError("Database restore", self) diff --git a/omop_alchemy/backends/postgres.py b/omop_alchemy/backends/postgres.py index 19ec50a..0c49933 100644 --- a/omop_alchemy/backends/postgres.py +++ b/omop_alchemy/backends/postgres.py @@ -1,26 +1,19 @@ from __future__ import annotations import os +import re import shutil import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import TSVECTOR +from oa_configurator import Dialect, Role, qualified, physical_schema_of +from sqlalchemy.dialects.postgresql import REGCONFIG, TSVECTOR from sqlalchemy.sql import func from .base import Backend, FullTextTargetConfig - -def _qualified(table_name: str, db_schema: str | None) -> str: - if db_schema: - return f'"{db_schema}"."{table_name}"' - return f'"{table_name}"' - - -def _qualified_index(index_name: str, db_schema: str | None) -> str: - if db_schema: - return f'"{db_schema}"."{index_name}"' - return f'"{index_name}"' +_STRING_LITERAL = re.compile(r"'(?:[^']|'')*'") +_TEXTLIKE_CAST = re.compile(r"::(?:text|varchar|character varying|bpchar|char)\b", re.IGNORECASE) class PostgresBackend(Backend): @@ -31,7 +24,7 @@ def name(self) -> str: @property def dialect(self) -> str: - return "postgresql" + return Dialect.POSTGRESQL # ── FK trigger management ──────────────────────────────────────────────── @@ -39,20 +32,21 @@ def toggle_fk_triggers( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, enable: bool, + schema_tag: str = Role.PRIMARY.value, ) -> None: action = "ENABLE" if enable else "DISABLE" conn.exec_driver_sql( - f"ALTER TABLE {_qualified(table_name, db_schema)} {action} TRIGGER ALL" + f"ALTER TABLE {qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))} {action} TRIGGER ALL" ) def get_fk_trigger_counts( self, conn: sa.Connection, table_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[int, int]: disabled_count, enabled_count = conn.execute( sa.text( @@ -69,7 +63,7 @@ def get_fk_trigger_counts( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": db_schema}, + {"table_name": table_name, "db_schema": physical_schema_of(conn, schema_tag=schema_tag)}, ).one() return int(disabled_count or 0), int(enabled_count or 0) @@ -80,10 +74,12 @@ def count_fk_violations( referred_table: str, constrained_cols: list[str], referred_cols: list[str], - db_schema: str | None, + *, + source_schema_tag: str = Role.PRIMARY.value, + referred_schema_tag: str = Role.PRIMARY.value, ) -> int: - source = _qualified(source_table, db_schema) - referred = _qualified(referred_table, db_schema) + source = qualified(conn, source_table, physical_schema=physical_schema_of(conn, schema_tag=source_schema_tag)) + referred = qualified(conn, referred_table, physical_schema=physical_schema_of(conn, schema_tag=referred_schema_tag)) non_null_predicate = " AND ".join( f"src.{col} IS NOT NULL" for col in constrained_cols ) @@ -113,17 +109,19 @@ def cluster_table( conn: sa.Connection, table_name: str, index_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> None: conn.exec_driver_sql( - f"CLUSTER {_qualified(table_name, db_schema)} USING {index_name}" + f"CLUSTER {qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))} USING {index_name}" ) def get_clustered_index_name( self, conn: sa.Connection, table_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> str | None: result = conn.execute( sa.text( @@ -138,30 +136,55 @@ def get_clustered_index_name( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": db_schema}, + {"table_name": table_name, "db_schema": physical_schema_of(conn, schema_tag=schema_tag)}, ).scalar_one_or_none() return str(result) if result is not None else None + # ── Schema reconciliation ──────────────────────────────────────────────── + + def normalize_index_expression(self, sql_text: str) -> str: + """Strip whitespace and casts to a text-ish type outside string + literals, and fold case the same way. + + Postgres's own catalog inserts these casts around string functions + as cosmetic noise when reflecting an index back, e.g. + ``lower(concept_name::text)``. A cast to any other type + (``::numeric``, ``::integer``, ...) is left intact, since that + changes the expression's actual computation. Literal contents are + never touched: identifiers, keywords, and casts are + case/whitespace-insensitive in Postgres, but a literal value isn't. + """ + parts = [] + last_end = 0 + for match in _STRING_LITERAL.finditer(sql_text): + before = sql_text[last_end : match.start()] + parts.append(_TEXTLIKE_CAST.sub("", before).replace(" ", "").lower()) + parts.append(match.group(0)) + last_end = match.end() + parts.append(_TEXTLIKE_CAST.sub("", sql_text[last_end:]).replace(" ", "").lower()) + return "".join(parts) + # ── Table operations ───────────────────────────────────────────────────── def analyze_table( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, vacuum: bool = False, + schema_tag: str = Role.PRIMARY.value, ) -> None: operation = "VACUUM ANALYZE" if vacuum else "ANALYZE" - conn.exec_driver_sql(f"{operation} {_qualified(table_name, db_schema)}") + conn.exec_driver_sql(f"{operation} {qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))}") def index_exists( self, conn: sa.Connection, index_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> bool: - qualified_index_name = _qualified_index(index_name, db_schema) + qualified_index_name = qualified(conn, index_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag)) return bool( conn.scalar( sa.select( @@ -170,20 +193,22 @@ def index_exists( ) ) - def drop_index_if_exists(self, conn: sa.Connection, index_name: str, db_schema: str | None) -> None: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {_qualified_index(index_name, db_schema)}") + def drop_index_if_exists( + self, conn: sa.Connection, index_name: str, *, schema_tag: str = Role.PRIMARY.value + ) -> None: + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))}") def truncate_table_batch( self, conn: sa.Connection, table_names: list[str], - db_schema: str | None, *, restart_identities: bool, cascade: bool, + schema_tag: str = Role.PRIMARY.value, ) -> None: sql = "TRUNCATE TABLE " + ", ".join( - _qualified(name, db_schema) for name in table_names + qualified(conn, name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag)) for name in table_names ) if restart_identities: sql += " RESTART IDENTITY" @@ -198,9 +223,10 @@ def find_sequence_name( conn: sa.Connection, table_name: str, column_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> str | None: - fully_qualified = _qualified(table_name, db_schema) + fully_qualified = qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag)) return conn.execute( sa.text("SELECT pg_get_serial_sequence(:table_name, :column_name)"), {"table_name": fully_qualified, "column_name": column_name}, @@ -217,28 +243,6 @@ def set_sequence_value( {"sequence_name": sequence_name, "value": value}, ) - # ── Schema context ─────────────────────────────────────────────────────── - - def configure_schema_context( - self, - conn: sa.Connection, - db_schema: str | None, - ) -> None: - if db_schema is None: - return - quoted = '"' + db_schema.replace('"', '""') + '"' - conn.exec_driver_sql(f"SET search_path TO {quoted}") - - def ensure_schema( - self, - conn: sa.Connection, - schema: str | None, - ) -> None: - if not schema or schema == "public": - return - quoted = '"' + schema.replace('"', '""') + '"' - conn.exec_driver_sql(f"CREATE SCHEMA IF NOT EXISTS {quoted}") - # ── Full-text search ───────────────────────────────────────────────────── @property @@ -308,20 +312,28 @@ def install_fulltext_on_table( table_name: str, vector_column_name: str, index_name: str, - db_schema: str | None, create_indexes: bool, fastupdate: bool, + schema_tag: str = Role.PRIMARY.value, ) -> None: - qualified_table = _qualified(table_name, db_schema) + qualified_table = qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag)) conn.exec_driver_sql( f"ALTER TABLE {qualified_table} ADD COLUMN IF NOT EXISTS {vector_column_name} tsvector" ) if create_indexes: - conn.exec_driver_sql( - f"CREATE INDEX IF NOT EXISTS {index_name}" - f" ON {qualified_table} USING GIN ({vector_column_name})" - f" WITH (fastupdate = {'on' if fastupdate else 'off'})" + lightweight_table = sa.Table( + table_name, + sa.MetaData(), + sa.Column(vector_column_name, TSVECTOR), + schema=physical_schema_of(conn, schema_tag=schema_tag), + ) + index = sa.Index( + index_name, + lightweight_table.c[vector_column_name], + postgresql_using="gin", + postgresql_with={"fastupdate": "on" if fastupdate else "off"}, ) + conn.execute(sa.schema.CreateIndex(index, if_not_exists=True)) def populate_fulltext_on_table( self, @@ -330,18 +342,25 @@ def populate_fulltext_on_table( table_name: str, vector_column_name: str, source_column_name: str, - db_schema: str | None, regconfig: str, + schema_tag: str = Role.PRIMARY.value, ) -> int | None: - result = conn.execute( - sa.text( - f"UPDATE {_qualified(table_name, db_schema)}" - f" SET {vector_column_name} = to_tsvector(" - f" CAST(:regconfig AS regconfig), coalesce({source_column_name}, '')" - f" )" - ), - {"regconfig": regconfig}, + lightweight_table = sa.table( + table_name, + sa.column(vector_column_name), + sa.column(source_column_name), + schema=physical_schema_of(conn, schema_tag=schema_tag), ) + source_column = lightweight_table.c[source_column_name] + stmt = lightweight_table.update().values( + **{ + vector_column_name: func.to_tsvector( + sa.cast(sa.bindparam("regconfig"), REGCONFIG), + func.coalesce(source_column, ""), + ) + } + ) + result = conn.execute(stmt, {"regconfig": regconfig}) if result.rowcount is None or result.rowcount < 0: return None return int(result.rowcount) @@ -353,13 +372,13 @@ def drop_fulltext_on_table( table_name: str, vector_column_name: str, index_name: str, - db_schema: str | None, drop_indexes: bool, + schema_tag: str = Role.PRIMARY.value, ) -> None: if drop_indexes: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {_qualified_index(index_name, db_schema)}") + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))}") conn.exec_driver_sql( - f"ALTER TABLE {_qualified(table_name, db_schema)}" + f"ALTER TABLE {qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))}" f" DROP COLUMN IF EXISTS {vector_column_name}" ) @@ -370,7 +389,8 @@ def prepare_backup( engine: sa.Engine, output_path: str, backup_format: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[str, list[str], dict[str, str], str]: tool_path = _pg_dump_path() url = engine.url @@ -389,6 +409,7 @@ def prepare_backup( "--no-owner", "--no-privileges", ] + db_schema = physical_schema_of(engine, schema_tag=schema_tag) if db_schema: command.extend(["--schema", db_schema]) env = os.environ.copy() @@ -401,7 +422,8 @@ def prepare_restore( engine: sa.Engine, input_path: str, backup_format: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[str, list[str], dict[str, str], str]: url = engine.url database_name = url.database @@ -410,6 +432,7 @@ def prepare_restore( "Database restore requires a database name in the configured engine URL." ) connection_uri = _libpq_connection_uri(url) + db_schema = physical_schema_of(engine, schema_tag=schema_tag) if backup_format == "custom": tool_path = _pg_restore_path() diff --git a/omop_alchemy/backends/resolve.py b/omop_alchemy/backends/resolve.py index f69e526..afea8c8 100644 --- a/omop_alchemy/backends/resolve.py +++ b/omop_alchemy/backends/resolve.py @@ -1,32 +1,33 @@ from __future__ import annotations -from enum import StrEnum import sqlalchemy as sa +from oa_configurator import Dialect from .base import Backend, BackendNotSupportedError from .postgres import PostgresBackend from .sqlite import SQLiteBackend - -class SupportedDialect(StrEnum): - POSTGRESQL = "postgresql" - SQLITE = "sqlite" - -_DIALECT_TO_BACKEND_MAP: dict[SupportedDialect, Backend] = { - SupportedDialect.POSTGRESQL: PostgresBackend(), - SupportedDialect.SQLITE: SQLiteBackend(), +_DIALECT_TO_BACKEND_MAP: dict[Dialect, Backend] = { + Dialect.POSTGRESQL: PostgresBackend(), + Dialect.SQLITE: SQLiteBackend(), } -def resolve_backend(engine: sa.Engine) -> Backend: +def resolve_backend(engine: sa.Engine | sa.Connection) -> Backend: dialect = engine.dialect.name try: - supported_dialect = SupportedDialect(dialect) + supported_dialect = Dialect(dialect) except ValueError: raise BackendNotSupportedError( f"Unsupported database dialect: '{dialect}'. " - f"Supported dialects: {', '.join(sorted(SupportedDialect))}." + f"Supported dialects: {', '.join(sorted(Dialect))}." ) return _DIALECT_TO_BACKEND_MAP[supported_dialect] - +def backend_label(dialect_name: str) -> str: + """Human-readable backend name for a dialect, falling back to the raw + dialect name for anything unrecognized (e.g. connection not yet resolved).""" + try: + return _DIALECT_TO_BACKEND_MAP[Dialect(dialect_name)].name + except (ValueError, KeyError): + return dialect_name diff --git a/omop_alchemy/backends/sqlite.py b/omop_alchemy/backends/sqlite.py index d584ca1..c7fb640 100644 --- a/omop_alchemy/backends/sqlite.py +++ b/omop_alchemy/backends/sqlite.py @@ -1,6 +1,7 @@ from __future__ import annotations import sqlalchemy as sa +from oa_configurator import Dialect, Role from .base import Backend, FeatureNotSupportedError @@ -13,13 +14,14 @@ def name(self) -> str: @property def dialect(self) -> str: - return "sqlite" + return Dialect.SQLITE def index_exists( self, conn: sa.Connection, index_name: str, - db_schema: str | None, + *, + schema_tag: str = Role.PRIMARY.value, ) -> bool: row = conn.exec_driver_sql( "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?", @@ -31,9 +33,9 @@ def analyze_table( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, vacuum: bool = False, + schema_tag: str = Role.PRIMARY.value, ) -> None: if vacuum: raise FeatureNotSupportedError("VACUUM ANALYZE", self) diff --git a/omop_alchemy/cdm/base/__init__.py b/omop_alchemy/cdm/base/__init__.py index 0c76798..6c703e3 100644 --- a/omop_alchemy/cdm/base/__init__.py +++ b/omop_alchemy/cdm/base/__init__.py @@ -1,9 +1,32 @@ from .cdm_table_base import CDMTableBase from .decorators import cdm_table, MODEL_MODULE_PREFIX -from .column_helpers import required_concept_fk, optional_concept_fk, optional_int, required_int -from .column_mixins import ValueMixin, ReferenceTable, DatedEvent, PersonScoped, HealthSystemContext, FactTable -from .indexing import merge_table_args, omop_index, omop_primary_key_index_name, omop_table_options -from .domain_validation import DomainValidationMixin, DomainRule, ExpectedDomain +from .column_helpers import ( + required_concept_fk, + optional_concept_fk, + role_fk, + role_table, + optional_int, + required_int +) +from .column_mixins import ( + ValueMixin, + ReferenceTable, + DatedEvent, + PersonScoped, + HealthSystemContext, + FactTable +) +from .indexing import ( + merge_table_args, + omop_index, + omop_primary_key_index_name, + omop_table_options +) +from .domain_validation import ( + DomainValidationMixin, + DomainRule, + ExpectedDomain +) from .concept_validation import ConceptValidationMixin from .reference_context import ReferenceContext from .typing import HasConceptId, HasEpisodeId, HasPersonId, DomainSemanticTable @@ -18,6 +41,8 @@ "MODEL_MODULE_PREFIX", "required_concept_fk", "optional_concept_fk", + "role_fk", + "role_table", "optional_int", "required_int", "ValueMixin", diff --git a/omop_alchemy/cdm/base/column_helpers.py b/omop_alchemy/cdm/base/column_helpers.py index 84d3480..e01689e 100644 --- a/omop_alchemy/cdm/base/column_helpers.py +++ b/omop_alchemy/cdm/base/column_helpers.py @@ -1,5 +1,51 @@ +from typing import Any import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role + + +def role_fk(role: Role, target: str) -> str: + """Schema-qualify an FK target string with a schema role placeholder. + + FK string resolution happens lazily, at mapper-configuration time, so + the target table's own schema role can't be looked up dynamically + without an import-order dependency on whichever file declares it; + role must be given explicitly, matching what the target table + declares in its own __table_args__. + + Parameters + ---------- + role : Role + Schema role the target table is tagged with. + target : str + Unqualified "table.column" FK target string. + + Returns + ------- + str + Schema-qualified FK target string. + """ + return f"{role.value}.{target}" + + +def role_table(role: Role, table: str) -> str: + """Schema-qualify a bare table-name string, e.g. for ``relationship(secondary=...)``. + If a target table is schema-qualified, its metadata key is ``role.table``. + + Parameters + ---------- + role : Role + Schema role the target table is tagged with. + table : str + Unqualified table name. + + Returns + ------- + str + Schema-qualified table-reference string. + """ + return f"{role.value}.{table}" + def required_concept_fk(): """ @@ -15,7 +61,7 @@ def required_concept_fk(): - Must exist - Unknown allowed (concept_id = 0) - Matches CDM Field-Level spec - - foreign key to `concept.concept_id` + - foreign key to `concept.concept_id`, always in the Role.VOCAB schema To index this column, add an explicit `omop_index(...)` to the model's `__table_args__` rather than indexing the column directly — @@ -23,25 +69,28 @@ def required_concept_fk(): """ return so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False, default=0, ) -def optional_concept_fk(): +def optional_concept_fk(**kwargs: Any): """ *optional_concept_fk* Used when a concept reference is genuinely optional. + foreign key to `concept.concept_id`, always in the Role.VOCAB schema. + To index this column, add an explicit `omop_index(...)` to the model's `__table_args__` rather than indexing the column directly — see `omop_alchemy.cdm.base.indexing`. """ return so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=True, + **kwargs, ) def optional_fk(target: str): @@ -50,6 +99,9 @@ def optional_fk(target: str): Optional foreign keys to non-concept tables. + target must already be schema-qualified (see role_fk()) if it points + into a Role-tagged table. + To index this column, add an explicit `omop_index(...)` to the model's `__table_args__` rather than indexing the column directly — see `omop_alchemy.cdm.base.indexing`. diff --git a/omop_alchemy/cdm/base/column_mixins.py b/omop_alchemy/cdm/base/column_mixins.py index eccb9a6..6ca2f82 100644 --- a/omop_alchemy/cdm/base/column_mixins.py +++ b/omop_alchemy/cdm/base/column_mixins.py @@ -4,6 +4,9 @@ from typing import Optional, Any import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role + +from .column_helpers import role_fk """ @@ -23,7 +26,7 @@ class PersonScoped: Encodes the standard `person_id` foreign key and indexing pattern. """ - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) class ConceptTyped: """ @@ -60,7 +63,7 @@ class ValueMixin: This helps when building generic tooling that needs to handle values flexibly but then normalise for analysis. """ value_as_number: so.Mapped[Optional[float]] = so.mapped_column(sa.Float, nullable=True) - value_as_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + value_as_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=True) class DatedEvent: """ @@ -87,9 +90,9 @@ class HealthSystemContext: Used across many clinical event tables to provide consistent join points into the health system structure. """ - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id"), nullable=True) - visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_occurrence.visit_occurrence_id"), nullable=True) - visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id"), nullable=True) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id")), nullable=True) + visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id")), nullable=True) + visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id")), nullable=True) class FactTable: """ @@ -116,7 +119,7 @@ class SourceAttribution: Mixin for *_source_value and *_source_concept_id patterns. """ source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String, nullable=True) - source_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + source_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=True) class UnitConcept: """ @@ -124,4 +127,4 @@ class UnitConcept: Mixin for unit_concept_id. """ - unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=True) diff --git a/omop_alchemy/cdm/base/concept_validation.py b/omop_alchemy/cdm/base/concept_validation.py index 92ced1f..09ae95f 100644 --- a/omop_alchemy/cdm/base/concept_validation.py +++ b/omop_alchemy/cdm/base/concept_validation.py @@ -8,7 +8,9 @@ class ConceptValidationMixin: A concept-bearing column is defined as: - column name ends with '_concept_id' - - value is integer-like + - column name does not contain 'source' (excludes *_source_concept_id) + + No type check is performed; matching is by column-name pattern only. Works for: - ORM mapped tables diff --git a/omop_alchemy/cdm/base/decorators.py b/omop_alchemy/cdm/base/decorators.py index b8e6247..b711be0 100644 --- a/omop_alchemy/cdm/base/decorators.py +++ b/omop_alchemy/cdm/base/decorators.py @@ -1,4 +1,7 @@ from typing import TypeVar + +from oa_configurator import validate_schema_tag + from .cdm_table_base import CDMTableBase T = TypeVar("T", bound=type) @@ -18,6 +21,7 @@ def cdm_table(cls: T) -> T: - Forces __abstract__ = False - Ensures __tablename__ is defined - Inherits from CDMTableBase + - Validates the table's own schema tag (see validate_schema_tag) - Used to clearly distinguish real CDM tables from mixins """ @@ -33,6 +37,16 @@ def cdm_table(cls: T) -> T: f"{cls.__name__} must inherit from CDMTableBase " ) + try: + schema_tag = validate_schema_tag(cls.__table__) # ty: ignore[unresolved-attribute] + except ValueError as exc: + raise TypeError(f"@cdm_table on {cls.__name__}: {exc}") from exc + if schema_tag is None: + raise TypeError( + f"@cdm_table on {cls.__name__}: table has no schema tag. " + "Every CDM table must declare __table_args__['schema']." + ) + # Explicitly mark as concrete cls.__abstract__ = False cls.__omop_is_cdm_table__ = True diff --git a/omop_alchemy/cdm/model/__init__.py b/omop_alchemy/cdm/model/__init__.py index fcd394f..6006d78 100644 --- a/omop_alchemy/cdm/model/__init__.py +++ b/omop_alchemy/cdm/model/__init__.py @@ -1,12 +1,55 @@ # we have to import at least one model from each module to ensure they are registered -from .clinical import Person, Condition_Occurrence, Death, Device_Exposure, Drug_Exposure, Measurement, Observation, Procedure_Occurrence -from .derived import Observation_Period, Condition_Era, Drug_Era, Dose_Era, Cohort_Definition, Cohort -from .vocabulary import Concept, Concept_Ancestor, Domain, Vocabulary, Concept_Class, Relationship, Concept_Relationship -from .health_system import Visit_Occurrence, Care_Site, Location, Provider, Visit_Detail -from .health_economic import Cost, Payer_Plan_Period -from .structural import Episode, Episode_Event, Fact_Relationship -from .unstructured import Note, Note_NLP -from .metadata import CDM_Source, Metadata +from .clinical import ( + Person, + Condition_Occurrence, + Death, + Device_Exposure, + Drug_Exposure, + Measurement, + Observation, + Observation_Period, + Procedure_Occurrence +) +from .derived import ( + Condition_Era, + Drug_Era, + Dose_Era, + Cohort_Definition, + Cohort +) +from .vocabulary import ( + Concept, + Concept_Ancestor, + Domain, + Vocabulary, + Concept_Class, + Relationship, + Concept_Relationship +) +from .health_system import ( + Visit_Occurrence, + Care_Site, + Location, + Provider, + Visit_Detail +) +from .health_economic import ( + Cost, + Payer_Plan_Period +) +from .structural import ( + Episode, + Episode_Event, + Fact_Relationship +) +from .unstructured import ( + Note, + Note_NLP +) +from .metadata import ( + CDM_Source, + Metadata +) from .flags import ( InvalidReasonMixin, InvalidReasonFlag, diff --git a/omop_alchemy/cdm/model/clinical/__init__.py b/omop_alchemy/cdm/model/clinical/__init__.py index ab59a0f..dfab685 100644 --- a/omop_alchemy/cdm/model/clinical/__init__.py +++ b/omop_alchemy/cdm/model/clinical/__init__.py @@ -6,6 +6,7 @@ from .drug_exposure import Drug_Exposure, Drug_ExposureContext, Drug_ExposureView from .measurement import Measurement, MeasurementContext, MeasurementView from .observation import Observation, ObservationContext, ObservationView +from .observation_period import Observation_Period from .person import Person, PersonView from .procedure_occurrence import ( Procedure_Occurrence, @@ -22,6 +23,7 @@ "Condition_Occurrence", "Condition_OccurrenceContext", "Condition_OccurrenceView", + "Observation_Period", "Drug_Exposure", "Drug_ExposureContext", "Drug_ExposureView", diff --git a/omop_alchemy/cdm/model/clinical/condition_occurrence.py b/omop_alchemy/cdm/model/clinical/condition_occurrence.py index bba3e43..42acf07 100644 --- a/omop_alchemy/cdm/model/clinical/condition_occurrence.py +++ b/omop_alchemy/cdm/model/clinical/condition_occurrence.py @@ -3,18 +3,21 @@ from sqlalchemy.ext.declarative import declared_attr from typing import Optional, TYPE_CHECKING from datetime import date, datetime +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( - PersonScoped, - HealthSystemContext, - FactTable, + PersonScoped, + HealthSystemContext, + FactTable, ReferenceContext, CDMTableBase, - cdm_table, + cdm_table, ModifierFieldConcepts, ClinicalEventMixin, merge_table_args, omop_index, + optional_concept_fk, + role_fk, ) if TYPE_CHECKING: @@ -31,22 +34,23 @@ class Condition_Occurrence( ): __tablename__ = "condition_occurrence" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "condition_concept_id"), omop_index(__tablename__, "visit_occurrence_id") ) condition_occurrence_id: so.Mapped[int] = so.mapped_column(primary_key=True) - condition_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + condition_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) condition_start_date: so.Mapped[date] = so.mapped_column(nullable=False) condition_start_datetime: so.Mapped[Optional[datetime]] = so.mapped_column() condition_end_date: so.Mapped[Optional[date]] = so.mapped_column() condition_end_datetime: so.Mapped[Optional[datetime]] = so.mapped_column() - condition_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + condition_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) stop_reason: so.Mapped[Optional[str]] = so.mapped_column(sa.String(20)) condition_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50)) condition_status_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50)) - condition_source_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) - condition_status_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) + condition_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() + condition_status_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() class Condition_OccurrenceContext(ReferenceContext): condition_concept: so.Mapped["Concept"] = ReferenceContext._reference_relationship(target="Concept", local_fk="condition_concept_id") # type: ignore[assignment] @@ -70,6 +74,8 @@ class Condition_OccurrenceView( ClinicalEventMixin ): __tablename__ = "condition_occurrence" + # Must match Condition_Occurrence's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "condition_occurrence_id" __concept_id_col__ = "condition_concept_id" diff --git a/omop_alchemy/cdm/model/clinical/death.py b/omop_alchemy/cdm/model/clinical/death.py index 602af29..2f740d8 100644 --- a/omop_alchemy/cdm/model/clinical/death.py +++ b/omop_alchemy/cdm/model/clinical/death.py @@ -1,10 +1,12 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, CDMTableBase, cdm_table, optional_concept_fk, @@ -23,9 +25,10 @@ class Death(CDMTableBase, Base): __tablename__ = "death" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_table_options(cluster_on=omop_primary_key_index_name("death")), ) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), primary_key=True) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), primary_key=True) death_date: so.Mapped[date] = so.mapped_column(nullable=False) death_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) death_type_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() @@ -41,6 +44,8 @@ class DeathContext(ReferenceContext): class DeathView(Death, DeathContext, DomainValidationMixin): __tablename__ = "death" + # Must match Death's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { diff --git a/omop_alchemy/cdm/model/clinical/device_exposure.py b/omop_alchemy/cdm/model/clinical/device_exposure.py index 4a77e7d..f2f0286 100644 --- a/omop_alchemy/cdm/model/clinical/device_exposure.py +++ b/omop_alchemy/cdm/model/clinical/device_exposure.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date, datetime @@ -37,6 +38,7 @@ class Device_Exposure( ): __tablename__ = "device_exposure" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "device_concept_id"), omop_index(__tablename__, "visit_occurrence_id"), @@ -122,6 +124,8 @@ class Device_ExposureView( """Analytical Device Exposure mapping with event metadata and references.""" __tablename__ = "device_exposure" + # Must match Device_Exposure's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "device_exposure_id" __concept_id_col__ = "device_concept_id" diff --git a/omop_alchemy/cdm/model/clinical/drug_exposure.py b/omop_alchemy/cdm/model/clinical/drug_exposure.py index 621401d..bdde5bb 100644 --- a/omop_alchemy/cdm/model/clinical/drug_exposure.py +++ b/omop_alchemy/cdm/model/clinical/drug_exposure.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date, datetime from orm_loader.helpers import Base @@ -32,6 +33,7 @@ class Drug_Exposure( ): __tablename__ = "drug_exposure" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), omop_index(__tablename__, "visit_occurrence_id") @@ -76,6 +78,8 @@ class Drug_ExposureView( ): __tablename__ = "drug_exposure" + # Must match Drug_Exposure's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "drug_exposure_id" diff --git a/omop_alchemy/cdm/model/clinical/measurement.py b/omop_alchemy/cdm/model/clinical/measurement.py index 3342d86..82bc2c3 100644 --- a/omop_alchemy/cdm/model/clinical/measurement.py +++ b/omop_alchemy/cdm/model/clinical/measurement.py @@ -4,6 +4,7 @@ import sqlalchemy.orm as so from typing import Optional, TYPE_CHECKING from datetime import date, datetime +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( CDMTableBase, @@ -14,6 +15,8 @@ ClinicalEventMixin, ReferenceContext, cdm_table, + optional_concept_fk, + role_fk, ValueMixin, merge_table_args, omop_index, @@ -29,6 +32,7 @@ class Measurement(Base, CDMTableBase, ValueMixin, ModifierSourceMixin): __tablename__ = "measurement" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "measurement_concept_id"), omop_index(__tablename__, "visit_occurrence_id"), @@ -37,53 +41,30 @@ class Measurement(Base, CDMTableBase, ValueMixin, ModifierSourceMixin): ) measurement_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("person.person_id"), nullable=False - ) - measurement_concept_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), nullable=False - ) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) + measurement_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) measurement_date: so.Mapped[date] = so.mapped_column(nullable=False) measurement_datetime: so.Mapped[Optional[datetime]] measurement_time: so.Mapped[Optional[str]] - measurement_type_concept_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), nullable=False - ) - operator_concept_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("concept.concept_id") - ) - unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("concept.concept_id") - ) + measurement_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) + operator_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() + unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() range_low: so.Mapped[Optional[float]] range_high: so.Mapped[Optional[float]] - provider_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("provider.provider_id") - ) - visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("visit_occurrence.visit_occurrence_id") - ) - visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("visit_detail.visit_detail_id") - ) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id"))) + visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id"))) + visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id"))) measurement_source_value: so.Mapped[Optional[str]] - measurement_source_concept_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("concept.concept_id") - ) + measurement_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() unit_source_value: so.Mapped[Optional[str]] - unit_source_concept_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("concept.concept_id") - ) + unit_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() value_source_value: so.Mapped[Optional[str]] measurement_event_id: so.Mapped[Optional[int]] - meas_event_field_concept_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), - doc="Identifies which OMOP table measurement_event_id refers to", - ) + meas_event_field_concept_id: so.Mapped[Optional[int]] = optional_concept_fk(doc="Identifies which OMOP table measurement_event_id refers to") __modifier_event_id_col__ = "measurement_event_id" __modifier_field_concept_id_col__ = "meas_event_field_concept_id" @@ -145,6 +126,8 @@ class MeasurementView( """Analytical Measurement mapping with event metadata and reference context.""" __tablename__ = "measurement" + # Must match Measurement's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "measurement_id" __concept_id_col__ = "measurement_concept_id" diff --git a/omop_alchemy/cdm/model/clinical/observation.py b/omop_alchemy/cdm/model/clinical/observation.py index abd1ee8..4fdc7ee 100644 --- a/omop_alchemy/cdm/model/clinical/observation.py +++ b/omop_alchemy/cdm/model/clinical/observation.py @@ -4,6 +4,7 @@ import sqlalchemy.orm as so from typing import Optional, TYPE_CHECKING from datetime import date, datetime +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( CDMTableBase, @@ -14,6 +15,8 @@ ClinicalEventMixin, ReferenceContext, cdm_table, + optional_concept_fk, + role_fk, ValueMixin, merge_table_args, omop_index, @@ -29,6 +32,7 @@ class Observation(Base, CDMTableBase, ValueMixin, ModifierSourceMixin): __tablename__ = "observation" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "observation_concept_id"), omop_index(__tablename__, "visit_occurrence_id"), @@ -36,46 +40,26 @@ class Observation(Base, CDMTableBase, ValueMixin, ModifierSourceMixin): ) observation_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("person.person_id"), nullable=False - ) - observation_concept_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), nullable=False - ) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) + observation_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) observation_date: so.Mapped[date] = so.mapped_column(nullable=False) observation_datetime: so.Mapped[Optional[datetime]] - observation_type_concept_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), nullable=False - ) - # value_as_number: so.Mapped[Optional[float]] + observation_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) + #value_as_number: so.Mapped[Optional[float]] value_as_string: so.Mapped[Optional[str]] - # value_as_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) - qualifier_concept_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("concept.concept_id") - ) - unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("concept.concept_id") - ) - provider_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("provider.provider_id") - ) - visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("visit_occurrence.visit_occurrence_id") - ) - visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("visit_detail.visit_detail_id") - ) + #value_as_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() + qualifier_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() + unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id"))) + visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id"))) + visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id"))) observation_source_value: so.Mapped[Optional[str]] - observation_source_concept_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("concept.concept_id") - ) + observation_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() unit_source_value: so.Mapped[Optional[str]] qualifier_source_value: so.Mapped[Optional[str]] value_source_value: so.Mapped[Optional[str]] observation_event_id: so.Mapped[Optional[int]] - obs_event_field_concept_id: so.Mapped[Optional[int]] = so.mapped_column( - sa.ForeignKey("concept.concept_id") - ) + obs_event_field_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() __modifier_event_id_col__ = "observation_event_id" __modifier_field_concept_id_col__ = "obs_event_field_concept_id" @@ -131,6 +115,8 @@ class ObservationView( """Analytical Observation mapping with event metadata and reference context.""" __tablename__ = "observation" + # Must match Observation's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "observation_id" __concept_id_col__ = "observation_concept_id" diff --git a/omop_alchemy/cdm/model/derived/observation_period.py b/omop_alchemy/cdm/model/clinical/observation_period.py similarity index 79% rename from omop_alchemy/cdm/model/derived/observation_period.py rename to omop_alchemy/cdm/model/clinical/observation_period.py index b0d7bcc..adac764 100644 --- a/omop_alchemy/cdm/model/derived/observation_period.py +++ b/omop_alchemy/cdm/model/clinical/observation_period.py @@ -1,8 +1,10 @@ import sqlalchemy as sa import sqlalchemy.orm as so from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -16,10 +18,11 @@ class Observation_Period(CDMTableBase, Base): __table_args__ = merge_table_args( omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "period_type_concept_id"), + {"schema": Role.PRIMARY.value}, ) observation_period_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) observation_period_start_date: so.Mapped[date] = so.mapped_column(nullable=False) observation_period_end_date: so.Mapped[date] = so.mapped_column(nullable=False) period_type_concept_id: so.Mapped[int] = required_concept_fk() diff --git a/omop_alchemy/cdm/model/clinical/person.py b/omop_alchemy/cdm/model/clinical/person.py index d5c5153..8e3d850 100644 --- a/omop_alchemy/cdm/model/clinical/person.py +++ b/omop_alchemy/cdm/model/clinical/person.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from sqlalchemy.ext.declarative import declared_attr from typing import Optional from datetime import date @@ -9,6 +10,7 @@ from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -28,12 +30,13 @@ from ..vocabulary import Concept from ..health_system import Location, Provider, Care_Site from .death import Death -from ..derived import Observation_Period +from .observation_period import Observation_Period @cdm_table class Person(CDMTableBase,Base,HealthSystemContext): __tablename__ = "person" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "gender_concept_id"), omop_table_options(cluster_on=omop_primary_key_index_name("person")), ) @@ -52,9 +55,9 @@ class Person(CDMTableBase,Base,HealthSystemContext): race_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() ethnicity_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() - location_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("location.location_id"), nullable=True) - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id"), nullable=True) - care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("care_site.care_site_id"), nullable=True) + location_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "location.location_id")), nullable=True) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id")), nullable=True) + care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "care_site.care_site_id")), nullable=True) person_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) gender_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) @@ -105,6 +108,8 @@ class PersonView(Person, PersonContext, DomainValidationMixin): Avoid in ETL loops. """ __tablename__ = "person" + # Must match Person's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { "gender_concept_id": ExpectedDomain("Gender"), diff --git a/omop_alchemy/cdm/model/clinical/procedure_occurrence.py b/omop_alchemy/cdm/model/clinical/procedure_occurrence.py index 4b4cbfa..6ece122 100644 --- a/omop_alchemy/cdm/model/clinical/procedure_occurrence.py +++ b/omop_alchemy/cdm/model/clinical/procedure_occurrence.py @@ -1,6 +1,7 @@ from __future__ import annotations import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date from orm_loader.helpers import Base @@ -29,6 +30,7 @@ class Procedure_Occurrence(CDMTableBase, Base, PersonScoped, HealthSystemContext): __tablename__ = "procedure_occurrence" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "procedure_concept_id"), omop_index(__tablename__, "visit_occurrence_id") @@ -108,6 +110,8 @@ class Procedure_OccurrenceView( ClinicalEventMixin, ): __tablename__ = "procedure_occurrence" + # Must match Procedure_Occurrence's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "procedure_occurrence_id" diff --git a/omop_alchemy/cdm/model/clinical/specimen.py b/omop_alchemy/cdm/model/clinical/specimen.py index f33a00e..9642d55 100644 --- a/omop_alchemy/cdm/model/clinical/specimen.py +++ b/omop_alchemy/cdm/model/clinical/specimen.py @@ -1,9 +1,11 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, CDMTableBase, cdm_table, required_concept_fk, @@ -16,12 +18,13 @@ class Specimen(CDMTableBase, Base): __tablename__ = "specimen" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "specimen_concept_id") ) specimen_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) specimen_concept_id: so.Mapped[int] = required_concept_fk() specimen_type_concept_id: so.Mapped[int] = required_concept_fk() diff --git a/omop_alchemy/cdm/model/derived/__init__.py b/omop_alchemy/cdm/model/derived/__init__.py index 4506634..550d714 100644 --- a/omop_alchemy/cdm/model/derived/__init__.py +++ b/omop_alchemy/cdm/model/derived/__init__.py @@ -3,13 +3,11 @@ from .condition_era import Condition_Era from .drug_era import Drug_Era from .dose_era import Dose_Era -from .observation_period import Observation_Period __all__ = [ - "Cohort_Definition", - "Cohort", - "Condition_Era", + "Cohort_Definition", + "Cohort", + "Condition_Era", "Drug_Era", - "Dose_Era", - "Observation_Period" + "Dose_Era", ] diff --git a/omop_alchemy/cdm/model/derived/cohort.py b/omop_alchemy/cdm/model/derived/cohort.py index 987584c..cf1544a 100644 --- a/omop_alchemy/cdm/model/derived/cohort.py +++ b/omop_alchemy/cdm/model/derived/cohort.py @@ -1,15 +1,18 @@ import sqlalchemy as sa import sqlalchemy.orm as so from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, CDMTableBase, + merge_table_args, ) @cdm_table class Cohort(CDMTableBase, Base): __tablename__ = "cohort" + __table_args__ = merge_table_args({"schema": Role.RESULTS.value}) cohort_definition_id: so.Mapped[int] = so.mapped_column(primary_key=True) subject_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/derived/cohort_definition.py b/omop_alchemy/cdm/model/derived/cohort_definition.py index 3cd8198..abddab9 100644 --- a/omop_alchemy/cdm/model/derived/cohort_definition.py +++ b/omop_alchemy/cdm/model/derived/cohort_definition.py @@ -2,29 +2,32 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, CDMTableBase, merge_table_args, omop_index, + role_fk, ) @cdm_table class Cohort_Definition(CDMTableBase, Base): __tablename__ = "cohort_definition" __table_args__ = merge_table_args( + {"schema": Role.RESULTS.value}, omop_index(__tablename__, "definition_type_concept_id"), - omop_index(__tablename__, "subject_concept_id") + omop_index(__tablename__, "subject_concept_id"), ) cohort_definition_id: so.Mapped[int] = so.mapped_column(primary_key=True) cohort_definition_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) cohort_definition_description: so.Mapped[Optional[str]] = so.mapped_column(sa.Text, nullable=True) - definition_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + definition_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) cohort_definition_syntax: so.Mapped[Optional[str]] = so.mapped_column(sa.Text, nullable=True) - subject_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + subject_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) cohort_initiation_date: so.Mapped[Optional[date]] = so.mapped_column(sa.Date, nullable=True) diff --git a/omop_alchemy/cdm/model/derived/condition_era.py b/omop_alchemy/cdm/model/derived/condition_era.py index 3891911..0d2b7ab 100644 --- a/omop_alchemy/cdm/model/derived/condition_era.py +++ b/omop_alchemy/cdm/model/derived/condition_era.py @@ -2,8 +2,10 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -15,12 +17,13 @@ class Condition_Era(CDMTableBase, Base): __tablename__ = "condition_era" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "condition_concept_id"), ) condition_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) condition_concept_id: so.Mapped[int] = required_concept_fk() condition_era_start_date: so.Mapped[date] = so.mapped_column(nullable=False) condition_era_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/derived/dose_era.py b/omop_alchemy/cdm/model/derived/dose_era.py index 85f0975..44cd213 100644 --- a/omop_alchemy/cdm/model/derived/dose_era.py +++ b/omop_alchemy/cdm/model/derived/dose_era.py @@ -1,8 +1,10 @@ import sqlalchemy as sa import sqlalchemy.orm as so from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -14,13 +16,14 @@ class Dose_Era(CDMTableBase, Base): __tablename__ = "dose_era" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), omop_index(__tablename__, "unit_concept_id"), ) dose_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) drug_concept_id: so.Mapped[int] = required_concept_fk() unit_concept_id: so.Mapped[int] = required_concept_fk() dose_value: so.Mapped[float] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/derived/drug_era.py b/omop_alchemy/cdm/model/derived/drug_era.py index 695c3ca..26a483e 100644 --- a/omop_alchemy/cdm/model/derived/drug_era.py +++ b/omop_alchemy/cdm/model/derived/drug_era.py @@ -2,8 +2,10 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -15,12 +17,13 @@ class Drug_Era(CDMTableBase, Base): __tablename__ = "drug_era" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), ) drug_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) drug_concept_id: so.Mapped[int] = required_concept_fk() drug_era_start_date: so.Mapped[date] = so.mapped_column(nullable=False) drug_era_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/health_economic/cost.py b/omop_alchemy/cdm/model/health_economic/cost.py index 71c12a6..97c383d 100644 --- a/omop_alchemy/cdm/model/health_economic/cost.py +++ b/omop_alchemy/cdm/model/health_economic/cost.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( @@ -15,6 +16,7 @@ class Cost(CDMTableBase, Base): __tablename__ = "cost" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "cost_event_id"), omop_index(__tablename__, "cost_type_concept_id"), ) diff --git a/omop_alchemy/cdm/model/health_economic/payer_plan_period.py b/omop_alchemy/cdm/model/health_economic/payer_plan_period.py index 61bd5cb..99df58b 100644 --- a/omop_alchemy/cdm/model/health_economic/payer_plan_period.py +++ b/omop_alchemy/cdm/model/health_economic/payer_plan_period.py @@ -1,9 +1,11 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, optional_concept_fk, @@ -15,11 +17,12 @@ class Payer_Plan_Period(CDMTableBase, Base): __tablename__ = "payer_plan_period" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), ) payer_plan_period_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) payer_plan_period_start_date: so.Mapped[date] = so.mapped_column(nullable=False) payer_plan_period_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/health_system/care_site.py b/omop_alchemy/cdm/model/health_system/care_site.py index ba5b868..ec6dee5 100644 --- a/omop_alchemy/cdm/model/health_system/care_site.py +++ b/omop_alchemy/cdm/model/health_system/care_site.py @@ -1,9 +1,11 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, optional_concept_fk, @@ -17,6 +19,7 @@ class Care_Site(CDMTableBase, Base): __tablename__ = "care_site" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "place_of_service_concept_id"), omop_index(__tablename__, "location_id"), omop_table_options(cluster_on=omop_primary_key_index_name("care_site")), @@ -25,7 +28,7 @@ class Care_Site(CDMTableBase, Base): care_site_id: so.Mapped[int] = so.mapped_column(primary_key=True) care_site_name: so.Mapped[Optional[str]] = so.mapped_column(sa.String(255), nullable=True) place_of_service_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() - location_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("location.location_id"), nullable=True) + location_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "location.location_id")), nullable=True) care_site_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) place_of_service_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) diff --git a/omop_alchemy/cdm/model/health_system/location.py b/omop_alchemy/cdm/model/health_system/location.py index 95e8c87..66732c3 100644 --- a/omop_alchemy/cdm/model/health_system/location.py +++ b/omop_alchemy/cdm/model/health_system/location.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from orm_loader.helpers import Base @@ -17,6 +18,7 @@ class Location(CDMTableBase, Base): __tablename__ = "location" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "country_concept_id"), omop_table_options(cluster_on=omop_primary_key_index_name("location")), ) diff --git a/omop_alchemy/cdm/model/health_system/provider.py b/omop_alchemy/cdm/model/health_system/provider.py index f2c6975..4462a83 100644 --- a/omop_alchemy/cdm/model/health_system/provider.py +++ b/omop_alchemy/cdm/model/health_system/provider.py @@ -1,8 +1,10 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, optional_concept_fk, @@ -16,6 +18,7 @@ class Provider(CDMTableBase, Base): __tablename__ = "provider" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "specialty_concept_id"), omop_index(__tablename__, "care_site_id"), omop_index(__tablename__, "gender_concept_id"), @@ -27,7 +30,7 @@ class Provider(CDMTableBase, Base): npi: so.Mapped[Optional[str]] = so.mapped_column(sa.String(20), nullable=True) dea: so.Mapped[Optional[str]] = so.mapped_column(sa.String(20), nullable=True) specialty_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() - care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("care_site.care_site_id"), nullable=True) + care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "care_site.care_site_id")), nullable=True) year_of_birth: so.Mapped[Optional[int]] = so.mapped_column(sa.Integer, nullable=True) gender_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() provider_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) diff --git a/omop_alchemy/cdm/model/health_system/visit_detail.py b/omop_alchemy/cdm/model/health_system/visit_detail.py index a961e4f..342ba9b 100644 --- a/omop_alchemy/cdm/model/health_system/visit_detail.py +++ b/omop_alchemy/cdm/model/health_system/visit_detail.py @@ -1,10 +1,12 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -17,30 +19,31 @@ class Visit_Detail(CDMTableBase, Base): __tablename__ = "visit_detail" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "visit_detail_concept_id"), omop_index(__tablename__, "visit_occurrence_id") ) visit_detail_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) visit_detail_concept_id: so.Mapped[int] = required_concept_fk() visit_detail_start_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) visit_detail_start_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) visit_detail_end_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) visit_detail_end_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) visit_detail_type_concept_id: so.Mapped[int] = required_concept_fk() - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id"), nullable=True) - care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("care_site.care_site_id"), nullable=True) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id")), nullable=True) + care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "care_site.care_site_id")), nullable=True) visit_detail_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) visit_detail_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() admitted_from_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() admitted_from_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) discharged_to_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() discharged_to_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) - preceding_visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id"), nullable=True) - parent_visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id"), nullable=True) - visit_occurrence_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("visit_occurrence.visit_occurrence_id"), nullable=False) + preceding_visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id")), nullable=True) + parent_visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id")), nullable=True) + visit_occurrence_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id")), nullable=False) def __repr__(self) -> str: return f"" diff --git a/omop_alchemy/cdm/model/health_system/visit_occurrence.py b/omop_alchemy/cdm/model/health_system/visit_occurrence.py index 409b022..6437f2d 100644 --- a/omop_alchemy/cdm/model/health_system/visit_occurrence.py +++ b/omop_alchemy/cdm/model/health_system/visit_occurrence.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date from sqlalchemy.ext.declarative import declared_attr @@ -8,11 +9,13 @@ from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, + role_table, cdm_table, CDMTableBase, ReferenceContext, required_concept_fk, - optional_concept_fk, + optional_concept_fk, DomainValidationMixin, ExpectedDomain, merge_table_args, @@ -31,12 +34,13 @@ class Visit_Occurrence(CDMTableBase, Base): __tablename__ = "visit_occurrence" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "visit_concept_id"), ) visit_occurrence_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) visit_concept_id: so.Mapped[int] = required_concept_fk() visit_start_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) @@ -45,8 +49,8 @@ class Visit_Occurrence(CDMTableBase, Base): visit_end_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) visit_type_concept_id: so.Mapped[int] = required_concept_fk() - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id"), nullable=True) - care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("care_site.care_site_id"), nullable=True) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id")), nullable=True) + care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "care_site.care_site_id")), nullable=True) visit_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) visit_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() @@ -55,7 +59,7 @@ class Visit_Occurrence(CDMTableBase, Base): discharged_to_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() discharged_to_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) - preceding_visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_occurrence.visit_occurrence_id"), nullable=True) + preceding_visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id")), nullable=True) def __repr__(self) -> str: return f"" @@ -70,7 +74,7 @@ class VisitContext(ReferenceContext): def procedure_providers(cls) -> so.Mapped[list["Provider"]]: return so.relationship( "Provider", - secondary="procedure_occurrence", + secondary=role_table(Role.PRIMARY, "procedure_occurrence"), primaryjoin="Visit_Occurrence.visit_occurrence_id == Procedure_Occurrence.visit_occurrence_id", secondaryjoin="Provider.provider_id == Procedure_Occurrence.provider_id", viewonly=True, @@ -81,7 +85,7 @@ def procedure_providers(cls) -> so.Mapped[list["Provider"]]: def observation_providers(cls) -> so.Mapped[list["Provider"]]: return so.relationship( "Provider", - secondary="observation", + secondary=role_table(Role.PRIMARY, "observation"), primaryjoin="Visit_Occurrence.visit_occurrence_id == Observation.visit_occurrence_id", secondaryjoin="Provider.provider_id == Observation.provider_id", viewonly=True, @@ -90,6 +94,8 @@ def observation_providers(cls) -> so.Mapped[list["Provider"]]: class VisitView(Visit_Occurrence, VisitContext, DomainValidationMixin): __tablename__ = "visit_occurrence" + # Must match Visit_Occurrence's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { "visit_concept_id": ExpectedDomain("Visit"), diff --git a/omop_alchemy/cdm/model/metadata/cdm_source.py b/omop_alchemy/cdm/model/metadata/cdm_source.py index c02ee60..2696223 100644 --- a/omop_alchemy/cdm/model/metadata/cdm_source.py +++ b/omop_alchemy/cdm/model/metadata/cdm_source.py @@ -2,6 +2,7 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( @@ -9,12 +10,14 @@ CDMTableBase, merge_table_args, omop_index, + role_fk, ) @cdm_table class CDM_Source(CDMTableBase, Base): __tablename__ = "cdm_source" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "cdm_version_concept_id") ) @@ -30,7 +33,7 @@ class CDM_Source(CDMTableBase, Base): cdm_release_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) cdm_version: so.Mapped[Optional[str]] = so.mapped_column(sa.String(10), nullable=True) - cdm_version_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + cdm_version_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) vocabulary_version: so.Mapped[str] = so.mapped_column(sa.String(20), nullable=False) diff --git a/omop_alchemy/cdm/model/metadata/metadata.py b/omop_alchemy/cdm/model/metadata/metadata.py index 14e052c..42f724c 100644 --- a/omop_alchemy/cdm/model/metadata/metadata.py +++ b/omop_alchemy/cdm/model/metadata/metadata.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base @@ -17,6 +18,7 @@ class Metadata(CDMTableBase, Base, ValueMixin): __tablename__ = "metadata" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "metadata_concept_id", cluster=True), omop_index(__tablename__, "metadata_type_concept_id"), ) diff --git a/omop_alchemy/cdm/model/structural/episode.py b/omop_alchemy/cdm/model/structural/episode.py index eba6cfe..090dde3 100644 --- a/omop_alchemy/cdm/model/structural/episode.py +++ b/omop_alchemy/cdm/model/structural/episode.py @@ -1,10 +1,12 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from sqlalchemy.ext.declarative import declared_attr from typing import ClassVar, Optional, TYPE_CHECKING, List from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -30,6 +32,7 @@ class Episode(CDMTableBase, Base, PersonScoped): __tablename__ = "episode" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "episode_concept_id"), # following indices are not specified in the cdm but are likely to be useful for query performance @@ -39,7 +42,7 @@ class Episode(CDMTableBase, Base, PersonScoped): ) episode_id: so.Mapped[int] = so.mapped_column(primary_key=True) - episode_parent_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("episode.episode_id"), nullable=True) + episode_parent_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "episode.episode_id")), nullable=True) episode_start_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) episode_start_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) @@ -112,6 +115,8 @@ class EpisodeView(Episode, EpisodeContext, DomainValidationMixin, ModifierTarget """ __tablename__ = "episode" + # Must match Episode's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "episode_id" __concept_id_col__ = "episode_concept_id" diff --git a/omop_alchemy/cdm/model/structural/episode_event.py b/omop_alchemy/cdm/model/structural/episode_event.py index f700aa7..644f5c0 100644 --- a/omop_alchemy/cdm/model/structural/episode_event.py +++ b/omop_alchemy/cdm/model/structural/episode_event.py @@ -2,6 +2,7 @@ import sqlalchemy.orm as so from typing import TYPE_CHECKING, Any, Type from functools import cached_property +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, @@ -11,6 +12,7 @@ ExpectedDomain, merge_table_args, omop_index, + role_fk, ) from omop_alchemy.cdm.model.clinical.event_metadata import ( CLINICAL_EVENT_TARGETS_BY_FIELD_CONCEPT_ID, @@ -29,17 +31,14 @@ def clear_episode_event_target_class_cache() -> None: class Episode_Event(CDMTableBase, Base): __tablename__ = "episode_event" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "episode_id", cluster=True), omop_index(__tablename__, "episode_event_field_concept_id"), ) - episode_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("episode.episode_id"), nullable=False, primary_key=True - ) - event_id: so.Mapped[int] = so.mapped_column(nullable=False, primary_key=True) - episode_event_field_concept_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), nullable=False, primary_key=True - ) + episode_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "episode.episode_id")),nullable=False,primary_key=True) + event_id: so.Mapped[int] = so.mapped_column(nullable=False,primary_key=True) + episode_event_field_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False,primary_key=True) def __repr__(self) -> str: return f"" @@ -65,6 +64,8 @@ class Episode_EventView(Episode_Event, Episode_EventContext, DomainValidationMix """ __tablename__ = "episode_event" + # Must match Episode_Event's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { diff --git a/omop_alchemy/cdm/model/structural/fact_relationship.py b/omop_alchemy/cdm/model/structural/fact_relationship.py index ed40449..7b12051 100644 --- a/omop_alchemy/cdm/model/structural/fact_relationship.py +++ b/omop_alchemy/cdm/model/structural/fact_relationship.py @@ -1,37 +1,40 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( CDMTableBase, - cdm_table, + cdm_table, merge_table_args, omop_index, + role_fk, ) @cdm_table class Fact_Relationship(CDMTableBase, Base): __tablename__ = "fact_relationship" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "domain_concept_id_1"), omop_index(__tablename__, "domain_concept_id_2"), omop_index(__tablename__, "relationship_concept_id"), ) domain_concept_id_1: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), primary_key=True, nullable=False, ) fact_id_1: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True, nullable=False) domain_concept_id_2: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), primary_key=True, nullable=False, ) fact_id_2: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True, nullable=False) relationship_concept_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), primary_key=True, nullable=False, ) diff --git a/omop_alchemy/cdm/model/unstructured/note.py b/omop_alchemy/cdm/model/unstructured/note.py index 48e9868..f5e12cf 100644 --- a/omop_alchemy/cdm/model/unstructured/note.py +++ b/omop_alchemy/cdm/model/unstructured/note.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date, datetime @@ -29,6 +30,7 @@ class Note(CDMTableBase, Base, PersonScoped, HealthSystemContext): __tablename__ = "note" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "note_type_concept_id"), omop_index(__tablename__, "visit_occurrence_id"), @@ -105,6 +107,8 @@ class NoteContext(ReferenceContext): class NoteView(Note, NoteContext, DomainValidationMixin): __tablename__ = "note" + # Must match Note's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { diff --git a/omop_alchemy/cdm/model/unstructured/note_nlp.py b/omop_alchemy/cdm/model/unstructured/note_nlp.py index 0308728..9d92143 100644 --- a/omop_alchemy/cdm/model/unstructured/note_nlp.py +++ b/omop_alchemy/cdm/model/unstructured/note_nlp.py @@ -1,11 +1,13 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, CDMTableBase, cdm_table, optional_concept_fk, @@ -18,6 +20,7 @@ class Note_NLP(CDMTableBase, Base): __tablename__ = "note_nlp" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "note_id", cluster=True), omop_index(__tablename__, "note_nlp_concept_id"), ) @@ -25,7 +28,7 @@ class Note_NLP(CDMTableBase, Base): note_nlp_id: so.Mapped[int] = so.mapped_column(primary_key=True) note_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("note.note_id"), + sa.ForeignKey(role_fk(Role.PRIMARY, "note.note_id")), nullable=False, ) diff --git a/omop_alchemy/cdm/model/vocabulary/concept.py b/omop_alchemy/cdm/model/vocabulary/concept.py index 89af3e0..0517157 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept.py +++ b/omop_alchemy/cdm/model/vocabulary/concept.py @@ -10,6 +10,7 @@ from .concept_ancestor import Concept_Ancestor from .concept_relationship import Concept_Relationship +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -20,6 +21,7 @@ omop_index, omop_primary_key_index_name, omop_table_options, + role_fk, ) from omop_alchemy.cdm.model.flags import ( StandardConceptFlag, @@ -37,6 +39,7 @@ class Concept( ): __tablename__ = "concept" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "concept_code"), omop_index(__tablename__, "vocabulary_id"), omop_index(__tablename__, "domain_id"), @@ -52,9 +55,9 @@ class Concept( ) concept_id: so.Mapped[int] = so.mapped_column(primary_key=True) concept_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) - domain_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("domain.domain_id"), nullable=False) - vocabulary_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("vocabulary.vocabulary_id"), nullable=False) - concept_class_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("concept_class.concept_class_id"), nullable=False) + domain_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "domain.domain_id")), nullable=False) + vocabulary_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "vocabulary.vocabulary_id")), nullable=False) + concept_class_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept_class.concept_class_id")), nullable=False) standard_concept: so.Mapped[Optional[str]] = so.mapped_column(sa.String(1), nullable=True) concept_code: so.Mapped[str] = so.mapped_column(sa.String(50), nullable=False) valid_start_date: so.Mapped[date] = so.mapped_column(sa.Date(), nullable=False) @@ -160,4 +163,6 @@ class ConceptView(Concept, ConceptContext): Avoid in tight loops or ETL paths. """ __tablename__ = "concept" + # Must match Concept's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.VOCAB.value} __mapper_args__ = {"concrete": False} diff --git a/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py b/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py index d933377..d3d8c37 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -7,17 +8,19 @@ CDMTableBase, merge_table_args, omop_index, + role_fk, ) @cdm_table class Concept_Ancestor(Base, ReferenceTable, CDMTableBase): __tablename__ = "concept_ancestor" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "ancestor_concept_id", cluster=True), omop_index(__tablename__, "descendant_concept_id"), ) - ancestor_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) - descendant_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) + ancestor_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) + descendant_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) min_levels_of_separation: so.Mapped[int] = so.mapped_column(nullable=False) max_levels_of_separation: so.Mapped[int] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/concept_class.py b/omop_alchemy/cdm/model/vocabulary/concept_class.py index c76faf5..816811c 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_class.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_class.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -8,17 +9,19 @@ merge_table_args, omop_primary_key_index_name, omop_table_options, + role_fk, ) @cdm_table class Concept_Class(Base, ReferenceTable, CDMTableBase): __tablename__ = "concept_class" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_table_options(cluster_on=omop_primary_key_index_name("concept_class")), ) concept_class_id: so.Mapped[str] = so.mapped_column(sa.String(20), primary_key=True) concept_class_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) - concept_class_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),nullable=False,) + concept_class_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False,) def __repr__(self): return f"" diff --git a/omop_alchemy/cdm/model/vocabulary/concept_relationship.py b/omop_alchemy/cdm/model/vocabulary/concept_relationship.py index 7b08f32..e6fe352 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_relationship.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_relationship.py @@ -1,6 +1,7 @@ import sqlalchemy as sa import sqlalchemy.orm as so from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -8,6 +9,7 @@ CDMTableBase, merge_table_args, omop_index, + role_fk, ) from omop_alchemy.cdm.model.flags import InvalidReasonMixin @@ -20,12 +22,13 @@ class Concept_Relationship( ): __tablename__ = "concept_relationship" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "concept_id_1", cluster=True), omop_index(__tablename__, "concept_id_2"), omop_index(__tablename__, "relationship_id"), ) - concept_id_1: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) - concept_id_2: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) - relationship_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("relationship.relationship_id"),primary_key=True) + concept_id_1: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) + concept_id_2: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) + relationship_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "relationship.relationship_id")),primary_key=True) valid_start_date: so.Mapped[date] = so.mapped_column(nullable=False) valid_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/concept_synonym.py b/omop_alchemy/cdm/model/vocabulary/concept_synonym.py index fd124fa..1553b7b 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_synonym.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_synonym.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -7,12 +8,14 @@ CDMTableBase, merge_table_args, omop_index, + role_fk, ) @cdm_table class Concept_Synonym(Base, ReferenceTable, CDMTableBase): __tablename__ = "concept_synonym" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "concept_id", cluster=True), # Has to be wrapped in func.lower() as that is the common query # as it prevents captialisation mismatches between query and data. @@ -22,6 +25,6 @@ class Concept_Synonym(Base, ReferenceTable, CDMTableBase): name="ix_concept_synonym_concept_synonym_name_lower", ), ) - concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) + concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) concept_synonym_name: so.Mapped[str] = so.mapped_column(sa.String(1000),primary_key=True) - language_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) + language_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) diff --git a/omop_alchemy/cdm/model/vocabulary/domain.py b/omop_alchemy/cdm/model/vocabulary/domain.py index db1eb31..3c7de50 100644 --- a/omop_alchemy/cdm/model/vocabulary/domain.py +++ b/omop_alchemy/cdm/model/vocabulary/domain.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -8,6 +9,7 @@ merge_table_args, omop_primary_key_index_name, omop_table_options, + role_fk, ) @cdm_table @@ -15,10 +17,11 @@ class Domain(Base, ReferenceTable, CDMTableBase): __tablename__ = "domain" __table_args__ = merge_table_args( omop_table_options(cluster_on=omop_primary_key_index_name("domain")), + {"schema": Role.VOCAB.value}, ) domain_id: so.Mapped[str] = so.mapped_column(sa.String(20), primary_key=True) domain_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) - domain_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),nullable=False) + domain_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False) def __repr__(self): return f'' diff --git a/omop_alchemy/cdm/model/vocabulary/drug_strength.py b/omop_alchemy/cdm/model/vocabulary/drug_strength.py index 668224d..10feb2f 100644 --- a/omop_alchemy/cdm/model/vocabulary/drug_strength.py +++ b/omop_alchemy/cdm/model/vocabulary/drug_strength.py @@ -2,6 +2,7 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -9,6 +10,8 @@ CDMTableBase, merge_table_args, omop_index, + optional_concept_fk, + role_fk, ) from omop_alchemy.cdm.model.flags import InvalidReasonMixin @@ -27,18 +30,19 @@ class Drug_Strength( """ __tablename__ = "drug_strength" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "drug_concept_id", cluster=True), omop_index(__tablename__, "ingredient_concept_id"), ) - drug_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) - ingredient_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) + drug_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) + ingredient_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) amount_value: so.Mapped[Optional[float]] = so.mapped_column(sa.Float, nullable=True) - amount_unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + amount_unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() numerator_value: so.Mapped[Optional[float]] = so.mapped_column(sa.Float, nullable=True) - numerator_unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + numerator_unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() denominator_value: so.Mapped[Optional[float]] = so.mapped_column(sa.Float, nullable=True) - denominator_unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + denominator_unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() box_size: so.Mapped[Optional[int]] = so.mapped_column(sa.Integer, nullable=True) valid_start_date: so.Mapped[date] = so.mapped_column(nullable=False) valid_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/relationship.py b/omop_alchemy/cdm/model/vocabulary/relationship.py index fd72e77..ba2c277 100644 --- a/omop_alchemy/cdm/model/vocabulary/relationship.py +++ b/omop_alchemy/cdm/model/vocabulary/relationship.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -8,6 +9,7 @@ merge_table_args, omop_primary_key_index_name, omop_table_options, + role_fk, ) from omop_alchemy.cdm.model.flags import BooleanFlag, normalised_flag_expr, normalised_flag @@ -16,13 +18,14 @@ class Relationship(Base, ReferenceTable, CDMTableBase): __tablename__ = "relationship" __table_args__ = merge_table_args( omop_table_options(cluster_on=omop_primary_key_index_name("relationship")), + {"schema": Role.VOCAB.value}, ) relationship_id: so.Mapped[str] = so.mapped_column(sa.String(20), primary_key=True) relationship_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) is_hierarchical: so.Mapped[str] = so.mapped_column(sa.String(1), nullable=False) defines_ancestry: so.Mapped[str] = so.mapped_column(sa.String(1), nullable=False) - reverse_relationship_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("relationship.relationship_id"),nullable=False) - relationship_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),nullable=False,) + reverse_relationship_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "relationship.relationship_id")),nullable=False) + relationship_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False,) def __repr__(self): return f"" diff --git a/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py b/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py index 3b3da12..bebc0b9 100644 --- a/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py +++ b/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py @@ -3,6 +3,7 @@ from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from orm_loader.registry import ValidationIssue from omop_alchemy.cdm.base import ( @@ -12,6 +13,7 @@ DatedEvent, merge_table_args, omop_index, + role_fk, ) from omop_alchemy.cdm.model.flags import InvalidReasonMixin @@ -32,6 +34,7 @@ class Source_To_Concept_Map( __tablename__ = "source_to_concept_map" __cdm_extra_checks__ = ["source_concept_id_range"] __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "target_concept_id", cluster=True), omop_index(__tablename__, "source_vocabulary_id"), omop_index(__tablename__, "target_vocabulary_id"), @@ -42,8 +45,8 @@ class Source_To_Concept_Map( source_concept_id: so.Mapped[int] = so.mapped_column(sa.Integer,primary_key=True,doc="0 or >= 2,000,000,000 for site-specific concepts") source_vocabulary_id: so.Mapped[str] = so.mapped_column(sa.String(20),primary_key=True) source_code_description: so.Mapped[Optional[str]] = so.mapped_column(sa.String(255), nullable=True) - target_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) - target_vocabulary_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("vocabulary.vocabulary_id"),nullable=False) + target_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) + target_vocabulary_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "vocabulary.vocabulary_id")),nullable=False) valid_start_date: so.Mapped[date] = so.mapped_column(nullable=False) valid_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/vocabulary.py b/omop_alchemy/cdm/model/vocabulary/vocabulary.py index 4efa788..d670872 100644 --- a/omop_alchemy/cdm/model/vocabulary/vocabulary.py +++ b/omop_alchemy/cdm/model/vocabulary/vocabulary.py @@ -13,6 +13,7 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -21,6 +22,7 @@ merge_table_args, omop_primary_key_index_name, omop_table_options, + role_fk, ) @cdm_table @@ -28,12 +30,13 @@ class Vocabulary(Base, ReferenceTable, CDMTableBase): __tablename__ = "vocabulary" __table_args__ = merge_table_args( omop_table_options(cluster_on=omop_primary_key_index_name("vocabulary")), + {"schema": Role.VOCAB.value}, ) vocabulary_id: so.Mapped[str] = so.mapped_column(sa.String(20), primary_key=True) vocabulary_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) vocabulary_reference: so.Mapped[Optional[str]] = so.mapped_column(sa.String(255), nullable=True) vocabulary_version: so.Mapped[Optional[str]] = so.mapped_column(sa.String(255), nullable=True) - vocabulary_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),nullable=False,) + vocabulary_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False,) def __repr__(self): return f"" diff --git a/omop_alchemy/config.py b/omop_alchemy/config.py index 09bb7d9..5a63545 100644 --- a/omop_alchemy/config.py +++ b/omop_alchemy/config.py @@ -12,8 +12,14 @@ ResolvedCDMDatabase, Role, load_stack_config, + register_reserved_schema, ) +# Guaranteed to be imported and registered if there is a config +MAINTENANCE_SCHEMA: str = "omop_alchemy_maintenance" + +register_reserved_schema(MAINTENANCE_SCHEMA, owner="omop_alchemy") + class OmopAlchemyConfig(PackageConfigBase): """oa-configurator config class for omop-alchemy, the CDM database owner. @@ -26,9 +32,18 @@ class OmopAlchemyConfig(PackageConfigBase): ---------- cdm_db : str Name of the ``[databases.*]`` entry holding the CDM database. - test_cdm_db : str, optional + test_cdm_db_pg : str, optional Name of the ``[databases.*]`` entry holding the test CDM database, - marked ``RefTo(CDMDatabaseConfig, is_test=True)``. + marked ``RefTo(CDMDatabaseConfig, is_test=True)``. Must resolve to a + real PostgreSQL connection; used for real integration testing of + Postgres-only behavior (FK triggers, catalog queries, ALTER, etc.). + test_cdm_db_sqlite : str, optional + Same shape as ``test_cdm_db_pg``, for tests that must always run + against SQLite specifically (dialect-behavior tests), regardless of + what ``test_cdm_db_pg`` happens to be configured to. Left + unconfigured by design in every environment, since + ``isolated_test_database(..., dialect="sqlite")`` provisions a + disposable instance with no config needed at all. Notes ----- @@ -40,9 +55,21 @@ class OmopAlchemyConfig(PackageConfigBase): extra_logging_namespaces: ClassVar[tuple[str, ...]] = ("orm_loader",) cdm_db: Annotated[str, RefTo(CDMDatabaseConfig)] = "cdm_db" - test_cdm_db: Annotated[ + test_cdm_db_pg: Annotated[ str | None, RefTo(CDMDatabaseConfig, is_test=True) - ] = None + ] = Field( + default=None, + description="Real PostgreSQL test CDM database, for Postgres-only integration testing.", + ) + test_cdm_db_sqlite: Annotated[ + str | None, RefTo(CDMDatabaseConfig, is_test=True) + ] = Field( + default=None, + description=( + "Disposable SQLite test database; left unconfigured by design " + "(isolated_test_database(..., dialect='sqlite') provisions one automatically)." + ), + ) athena_source_path: str | None = Field( default=None, @@ -83,41 +110,23 @@ def get_cdm_context() -> tuple[OmopAlchemyConfig, ResolvedCDMDatabase]: def vocabulary_identity(resolved: ResolvedCDMDatabase) -> str | None: """Stable identity for the vocabulary dataset ``resolved`` reads, or None. - Concept-set expansions are a function of the vocabulary, so caching them - against this identity means recreating an engine against the same dataset - reuses the expansion instead of re-running ``concept_ancestor`` traversals. - - Composed from the **vocab** role rather than the primary one, because - ``concept_ancestor`` is a vocabulary table. On any deployment that does not - configure a separate vocabulary target this resolves to the CDM database, so - it costs nothing today and stays correct if vocabulary routing is ever - honoured by the ORM. Do not "simplify" it to ``resolved.connection``. - - Uses ``safe_url``, the credential-redacted form, so no password reaches a - cache key. - - **Returns None wherever sharing would be unsafe, so every caller inherits - that judgement.** Exported precisely so that packages building their own - engines compose the identity the same way — two spellings of one dataset - would produce two cache entries that each look authoritative. That only works - if the safety conditions live here rather than at one call site. - - Two conditions yield None: - - *Split vocabulary target.* Vocabulary models use the primary logical schema, - and one SQLAlchemy engine cannot route tables to a second physical - connection, so a declared vocabulary target that differs from the primary is - not what the engine actually reads. Returning its identity would let two - different primary databases that name the same external vocabulary share - expansions — one database's concept sets served for another. Such a - deployment falls back to per-engine caching until ORM routing supports it. - - *Ephemeral database.* In-memory SQLite, where two engines built from - identical configuration are genuinely separate databases. - - Both cases are correct-but-unshared rather than wrong. + Caches concept-set expansions (``concept_ancestor`` traversals) across + engines reading the same vocabulary. Built from the VOCAB role, not + primary, since ``concept_ancestor`` is a vocabulary table; do not + simplify to ``resolved.connection``. Uses ``safe_url`` so no password + reaches the cache key. + + Returns None wherever sharing would be unsafe, so every caller inherits + that judgement instead of each composing its own identity: + - a split vocabulary target: schema_translate_map cannot route to a different + physical connection, so identity based on the declared target would not + match what the engine actually reads, or + - an ephemeral database: in-memory SQLite, where identically-configured engines are genuinely + separate databases. + + Both fall back to per-engine caching instead of being wrong. """ - vocab_target = resolved.connection_target(Role.VOCAB) + vocab_target = resolved.connection_for_role(Role.VOCAB) if ( vocab_target.safe_url != resolved.connection.safe_url diff --git a/omop_alchemy/maintenance/_cli_utils.py b/omop_alchemy/maintenance/_cli_utils.py index a907c9a..c99b757 100644 --- a/omop_alchemy/maintenance/_cli_utils.py +++ b/omop_alchemy/maintenance/_cli_utils.py @@ -8,36 +8,18 @@ from enum import StrEnum from typing import Any, Callable, TypeVar -import sqlalchemy as sa import typer -from orm_loader.backends import STAGING_SCHEMA +from oa_configurator import ResolvedCDMDatabase from sqlalchemy.exc import SQLAlchemyError from .tables import TableCategory from .ui import console, render_error, render_command_header -from ..backends import BackendNotSupportedError, resolve_backend +from ..backends import BackendNotSupportedError _F = TypeVar("_F", bound=Callable[..., Any]) -class ReservedSchema(StrEnum): - """Schema names reserved for OMOP_Alchemy/orm-loader internal bookkeeping. - A user-configured db_schema may never collide with one of these. - """ - - STAGING = STAGING_SCHEMA - MAINTENANCE = "omop_alchemy_maintenance" - - -def reject_reserved_schema(db_schema: str | None) -> None: - """Raise if db_schema collides with a schema name reserved for internal bookkeeping.""" - if db_schema in set(ReservedSchema): - raise RuntimeError( - f"db_schema cannot be {db_schema!r}: reserved for OMOP_Alchemy/orm-loader internal use." - ) - - class Severity(StrEnum): """Coarse-grained outcome classification shared by every maintenance command's status vocabulary. @@ -86,14 +68,12 @@ def __new__(cls, code: str, severity: Severity): def __init__(self, code: str, severity: Severity): self.severity = severity - # -- shared across every dry-run/apply-style domain -- + # shared across every dry-run/apply-style domain PLANNED = ("planned", Severity.INFO) APPLIED = ("applied", Severity.OK) SKIPPED = ("skipped", Severity.WARNING) - # -- domain-specific "applied" words (backup, index restore/capture, - # sequence reset, vocab load) -- same OK severity as APPLIED, kept as - # distinct words since the specific outcome is worth seeing at a glance -- + # domain-specific "applied" words CREATED = ("created", Severity.OK) LOADED = ("loaded", Severity.OK) RESET = ("reset", Severity.OK) @@ -103,15 +83,16 @@ def __init__(self, code: str, severity: Severity): PASSED = ("passed", Severity.OK) MATCHED = ("matched", Severity.OK) - # -- warnings: something worth a look, but not blocking -- + # warnings: something worth a look, but not blocking WARNING = ("warning", Severity.WARNING) LIMITED = ("limited", Severity.WARNING) DRIFTED = ("drifted", Severity.WARNING) - # -- informational: an intentionally supported state -- + # informational: an intentionally supported state RENAMED = ("renamed", Severity.INFO) - # -- errors/failures -- + # errors/failures + RELOCATED = ("relocated", Severity.ERROR) MISSING = ("missing", Severity.ERROR) UNEXPECTED = ("unexpected", Severity.ERROR) MISMATCH = ("mismatch", Severity.ERROR) @@ -122,11 +103,16 @@ def __init__(self, code: str, severity: Severity): @dataclass(frozen=True) class _ConnContext: - """Connection context derived from the oa_configurator resolved resource.""" - db_schema: str | None - engine_url: str = "" + """Connection context assembled once per CLI command. + + resource_name and athena_source come from OmopAlchemyConfig, not from + resolved. Everything else a command needs (schema, vocab/results + schema, test_only, a vocab engine) is available via resolved directly, + or via resolved.vocab_engine_for(engine), rather than duplicated here. + """ + resolved: ResolvedCDMDatabase resource_name: str = "" - athena_source: str | None = None # from OmopAlchemyConfig.athena_source_path + athena_source: str | None = None # ── Decorator ───────────────────────────────────────────────────────────────── @@ -155,19 +141,17 @@ def wrapper(**kwargs: Any) -> Any: try: from ..config import create_cdm_engine, get_cdm_context pkg_config, resolved = get_cdm_context() - reject_reserved_schema(resolved.schema_name) engine = create_cdm_engine(resolved) conn = _ConnContext( - db_schema=resolved.schema_name, - engine_url=engine.url.render_as_string(hide_password=True), + resolved=resolved, resource_name=pkg_config.cdm_db, athena_source=pkg_config.athena_source_path, ) console.print( render_command_header( command_name=command_name, - engine_url=conn.engine_url, - db_schema=conn.db_schema, + engine_url=engine.url.render_as_string(hide_password=True), + db_schema=resolved.schema_name, vocabulary_included=_vocab, mode_label=_mode, ) @@ -211,21 +195,6 @@ def wrapper(**kwargs: Any) -> Any: # ── Helpers ─────────────────────────────────────────────────────────────────── -def ensure_schema(engine: sa.Engine, schema: str | None) -> None: - """Create the schema in the database if it does not already exist. - - Delegates to the active backend so behaviour is correct for each dialect. - No-op when schema is None or ``"public"``, or on backends that don't - support named schemas (e.g. SQLite). - """ - if not schema or schema == "public": - return - backend = resolve_backend(engine) - with engine.connect() as conn: - backend.ensure_schema(conn, schema) - conn.commit() - - def handle_error(exc: Exception) -> None: if isinstance(exc, BackendNotSupportedError): console.print(render_error(f"Not supported: {exc}")) diff --git a/omop_alchemy/maintenance/cli_backup.py b/omop_alchemy/maintenance/cli_backup.py index 765be4c..fc9b3f8 100644 --- a/omop_alchemy/maintenance/cli_backup.py +++ b/omop_alchemy/maintenance/cli_backup.py @@ -12,7 +12,7 @@ import typer from ..backends import resolve_backend, require_backend_support, backend_support_note -from ._cli_utils import Status, dry_label, dry_status, omop_command, reject_reserved_schema +from ._cli_utils import Status, dry_label, dry_status, omop_command from .ui import ( console, render_backup_result, @@ -65,14 +65,13 @@ def create_database_backup( dry_run: bool = False, ) -> BackupResult: """Create a database backup artifact at output_path. Runs the subprocess unless dry_run is True.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "prepare_backup", "Database backup") resolved_output_path = Path(output_path) if output_path is not None else _default_output_path(backup_format) resolved_output_path = resolved_output_path.expanduser().resolve() tool_path, command, env, database_name = backend.prepare_backup( - engine, str(resolved_output_path), backup_format.value, db_schema + engine, str(resolved_output_path), backup_format.value ) if not dry_run: @@ -112,14 +111,13 @@ def restore_database_backup( dry_run: bool = False, ) -> BackupResult: """Restore a database backup. Runs the subprocess unless dry_run is True.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "prepare_restore", "Database restore") resolved_input_path = Path(input_path).expanduser().resolve() if not resolved_input_path.exists(): raise RuntimeError(f"Backup artifact not found: {resolved_input_path}") tool_path, command, env, database_name = backend.prepare_restore( - engine, str(resolved_input_path), backup_format.value, db_schema + engine, str(resolved_input_path), backup_format.value ) if not dry_run: @@ -177,7 +175,7 @@ def backup_database_command( engine, output_path=output_path, backup_format=backup_format, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, dry_run=dry_run, ) console.print(render_backup_result(result)) @@ -202,7 +200,7 @@ def restore_database_command( engine, input_path=input_path, backup_format=backup_format, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, dry_run=dry_run, ) console.print(render_restore_result(result)) diff --git a/omop_alchemy/maintenance/cli_foreign_keys.py b/omop_alchemy/maintenance/cli_foreign_keys.py index a17d19f..b42e333 100644 --- a/omop_alchemy/maintenance/cli_foreign_keys.py +++ b/omop_alchemy/maintenance/cli_foreign_keys.py @@ -7,8 +7,9 @@ import sqlalchemy as sa import typer +from oa_configurator import physical_schema_of from ..backends import Backend, resolve_backend, require_backend_support, backend_support_note -from ._cli_utils import Status, dry_label, dry_status, omop_command, reject_reserved_schema +from ._cli_utils import Status, dry_label, dry_status, omop_command from .tables import ( TableCategory, existing_maintenance_tables, @@ -32,6 +33,7 @@ class ForeignKeyBase: table_name: str category: TableCategory + schema_tag: str @dataclass(frozen=True) @@ -87,24 +89,25 @@ class ForeignKeyValidationReport: def _collect_fk_info( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, ) -> list[_FKTableInfo]: """Return all ORM-managed tables that participate in at least one FK relationship (outgoing or incoming).""" inspector = sa.inspect(engine) selected_tables = existing_maintenance_tables( - inspector, - db_schema=db_schema, + engine, vocabulary_included=vocabulary_included, ) - selected_names = {table.table_name for table in selected_tables} + tables_by_name = {table.table_name: table for table in selected_tables} + selected_names = set(tables_by_name) incoming_counts = {name: 0 for name in selected_names} outgoing_counts = {name: 0 for name in selected_names} for table_name in selected_names: - foreign_keys = inspector.get_foreign_keys(table_name, schema=db_schema) + foreign_keys = inspector.get_foreign_keys( + table_name, schema=physical_schema_of(engine, schema_tag=tables_by_name[table_name].schema_tag) + ) relevant_foreign_keys = [ foreign_key for foreign_key in foreign_keys @@ -128,6 +131,7 @@ def _collect_fk_info( _FKTableInfo( table_name=table.table_name, category=table.category, + schema_tag=table.schema_tag, outgoing_constraint_count=outgoing_count, incoming_constraint_count=incoming_count, ) @@ -140,7 +144,6 @@ def _collect_strict_validation_failures( connection: sa.Connection, backend: Backend, *, - db_schema: str | None, vocabulary_included: bool, ) -> dict[str, list[ForeignKeyConstraintViolation]]: """Query every FK constraint across selected tables and return a mapping of table name → violation list. @@ -150,18 +153,21 @@ def _collect_strict_validation_failures( """ inspector = sa.inspect(connection) selected_tables = existing_maintenance_tables( - inspector, - db_schema=db_schema, + connection, vocabulary_included=vocabulary_included, ) - selected_names = {table.table_name for table in selected_tables} + tables_by_name = {table.table_name: table for table in selected_tables} + selected_names = set(tables_by_name) failures: dict[str, list[ForeignKeyConstraintViolation]] = { table_name: [] for table_name in selected_names } for table_name in sorted(selected_names): - for foreign_key in inspector.get_foreign_keys(table_name, schema=db_schema): + source_schema_tag = tables_by_name[table_name].schema_tag + for foreign_key in inspector.get_foreign_keys( + table_name, schema=physical_schema_of(connection, schema_tag=source_schema_tag) + ): referred_table = foreign_key.get("referred_table") constrained_columns = foreign_key.get("constrained_columns") or [] referred_columns = foreign_key.get("referred_columns") or [] @@ -179,7 +185,8 @@ def _collect_strict_validation_failures( str(referred_table), list(constrained_columns), list(referred_columns), - db_schema, + source_schema_tag=source_schema_tag, + referred_schema_tag=tables_by_name[str(referred_table)].schema_tag, ) if violation_count == 0: @@ -221,7 +228,6 @@ def _fk_violation_detail( def validate_foreign_key_constraints( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, ) -> ForeignKeyValidationReport: """Count rows that violate each FK constraint and return a full per-table validation report.""" @@ -230,7 +236,6 @@ def validate_foreign_key_constraints( targets = _collect_fk_info( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) @@ -238,7 +243,6 @@ def validate_foreign_key_constraints( validation_failures = _collect_strict_validation_failures( connection, backend, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) @@ -253,6 +257,7 @@ def validate_foreign_key_constraints( ForeignKeyValidationResult( table_name=target.table_name, category=target.category, + schema_tag=target.schema_tag, outgoing_constraint_count=target.outgoing_constraint_count, incoming_constraint_count=target.incoming_constraint_count, violating_constraint_count=violating_constraint_count, @@ -280,19 +285,16 @@ def manage_foreign_key_triggers( engine: sa.Engine, *, enable: bool = False, - db_schema: str | None = None, vocabulary_included: bool = False, dry_run: bool = False, strict: bool = False, ) -> list[ForeignKeyManagementResult]: """Enable or disable RI trigger enforcement. With strict=True, aborts on any FK violation.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "toggle_fk_triggers", "FK trigger management") targets = _collect_fk_info( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) @@ -302,7 +304,6 @@ def manage_foreign_key_triggers( validation_failures = _collect_strict_validation_failures( connection, backend, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) if validation_failures: @@ -312,6 +313,7 @@ def manage_foreign_key_triggers( ForeignKeyManagementResult( table_name=target.table_name, category=target.category, + schema_tag=target.schema_tag, outgoing_constraint_count=target.outgoing_constraint_count, incoming_constraint_count=target.incoming_constraint_count, enable=enable, @@ -333,12 +335,15 @@ def manage_foreign_key_triggers( } for target in targets: if not dry_run: - backend.toggle_fk_triggers(connection, target.table_name, db_schema, enable=enable) + backend.toggle_fk_triggers( + connection, target.table_name, enable=enable, schema_tag=target.schema_tag + ) results.append( ForeignKeyManagementResult( table_name=target.table_name, category=target.category, + schema_tag=target.schema_tag, outgoing_constraint_count=target.outgoing_constraint_count, incoming_constraint_count=target.incoming_constraint_count, enable=enable, @@ -353,7 +358,6 @@ def manage_foreign_key_triggers( def collect_foreign_key_trigger_status( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, ) -> list[ForeignKeyStatusResult]: """Query pg_trigger to count disabled vs enabled RI triggers for each participating table.""" @@ -362,7 +366,6 @@ def collect_foreign_key_trigger_status( targets = _collect_fk_info( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) results: list[ForeignKeyStatusResult] = [] @@ -370,12 +373,13 @@ def collect_foreign_key_trigger_status( with engine.connect() as connection: for target in targets: disabled_count, enabled_count = backend.get_fk_trigger_counts( - connection, target.table_name, db_schema + connection, target.table_name, schema_tag=target.schema_tag ) results.append( ForeignKeyStatusResult( table_name=target.table_name, category=target.category, + schema_tag=target.schema_tag, disabled_trigger_count=disabled_count, enabled_trigger_count=enabled_count, outgoing_constraint_count=target.outgoing_constraint_count, @@ -417,7 +421,6 @@ def disable_foreign_keys_command( results = manage_foreign_key_triggers( engine, enable=False, - db_schema=conn.db_schema, vocabulary_included=vocabulary_included, dry_run=dry_run, strict=strict, @@ -454,7 +457,6 @@ def enable_foreign_keys_command( results = manage_foreign_key_triggers( engine, enable=True, - db_schema=conn.db_schema, vocabulary_included=vocabulary_included, dry_run=dry_run, strict=strict, @@ -479,7 +481,6 @@ def foreign_key_status_command( with console.status("Inspecting foreign key trigger status..."): results = collect_foreign_key_trigger_status( engine, - db_schema=conn.db_schema, vocabulary_included=vocabulary_included, ) console.print(render_foreign_key_status_results(results)) @@ -501,7 +502,6 @@ def foreign_key_validate_command( with console.status("Validating selected foreign key relationships..."): report = validate_foreign_key_constraints( engine, - db_schema=conn.db_schema, vocabulary_included=vocabulary_included, ) console.print(render_foreign_key_validation_results(report.results)) diff --git a/omop_alchemy/maintenance/cli_fulltext.py b/omop_alchemy/maintenance/cli_fulltext.py index 13b788c..f5a4993 100644 --- a/omop_alchemy/maintenance/cli_fulltext.py +++ b/omop_alchemy/maintenance/cli_fulltext.py @@ -2,22 +2,45 @@ from __future__ import annotations +from contextlib import ExitStack from dataclasses import dataclass from enum import StrEnum +from typing import cast import typer +import sqlalchemy as sa from sqlalchemy.engine import Engine +from oa_configurator import ResolvedCDMDatabase, guard_schema_provenance_for, validate_schema_tag from ..backends import backend_support_note as _backend_support_note from ..backends import resolve_backend, require_backend_support from ..backends.base import FullTextError -from ._cli_utils import Status, dry_label, dry_status, omop_command, reject_reserved_schema +from ..cdm.model.vocabulary.concept import Concept +from ..cdm.model.vocabulary.concept_synonym import Concept_Synonym +from ._cli_utils import Status, dry_label, dry_status, omop_command from .ui import ( console, render_fulltext_results, render_fulltext_summary, ) +_FULLTEXT_TARGET_TABLES: dict[str, sa.Table] = { + "concept": cast(sa.Table, Concept.__table__), + "concept_synonym": cast(sa.Table, Concept_Synonym.__table__), +} + + +def _schema_tag_for_target(table_name: str) -> str: + """The schema tag a fulltext target table's own declared schema names. + Resolves via validate_schema_tag() rather than hardcoding Role.VOCAB, + so a future non-vocab fulltext target resolves correctly. + """ + tag = validate_schema_tag(_FULLTEXT_TARGET_TABLES[table_name]) + if tag is None: + raise TypeError(f"{table_name}: table has no schema tag.") + return tag + + app = typer.Typer( help=f"Manage full-text search for OMOP vocabulary tables. {_backend_support_note('install_fulltext_on_table')}", rich_markup_mode="rich", @@ -51,29 +74,38 @@ class FullTextResult: def install_fulltext_columns( engine: Engine, *, - db_schema: str | None = None, create_indexes: bool = True, fastupdate: bool = False, dry_run: bool = False, + resolved: ResolvedCDMDatabase | None = None, ) -> tuple[FullTextResult, ...]: """Install tsvector sidecar columns (and optionally GIN indexes) on OMOP vocabulary tables.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "install_fulltext_on_table", "Full-text search") targets = backend.fulltext_targets try: if not dry_run: - with engine.begin() as connection: + tables_by_schema_tag: dict[str, list[sa.Table]] = {} + for cfg in targets: + tag = _schema_tag_for_target(cfg.table_name) + tables_by_schema_tag.setdefault(tag, []).append(_FULLTEXT_TARGET_TABLES[cfg.table_name]) + # One provenance guard per schema_tag (count only known at runtime); ExitStack defers every write until the block below succeeds. + with engine.begin() as connection, ExitStack() as guard_stack: + for schema_tag, tables in tables_by_schema_tag.items(): + # A same-named column/index could already exist under a drifted schema, attached to an unrelated table. + guard_stack.enter_context( + guard_schema_provenance_for(connection, resolved, schema_tag=schema_tag, tables=tables) + ) for cfg in targets: backend.install_fulltext_on_table( connection, table_name=cfg.table_name, vector_column_name=cfg.vector_column_name, index_name=cfg.index_name, - db_schema=db_schema, create_indexes=create_indexes, fastupdate=fastupdate, + schema_tag=_schema_tag_for_target(cfg.table_name), ) backend.register_fulltext_metadata() except FullTextError: @@ -105,12 +137,10 @@ def install_fulltext_columns( def populate_fulltext_columns( engine: Engine, *, - db_schema: str | None = None, regconfig: str = "english", dry_run: bool = False, ) -> tuple[FullTextResult, ...]: """Populate tsvector sidecar columns with pre-computed search vectors.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "populate_fulltext_on_table", "Full-text search") targets = backend.fulltext_targets @@ -125,8 +155,8 @@ def populate_fulltext_columns( table_name=cfg.table_name, vector_column_name=cfg.vector_column_name, source_column_name=cfg.source_column_name, - db_schema=db_schema, regconfig=regconfig, + schema_tag=_schema_tag_for_target(cfg.table_name), ) backend.register_fulltext_metadata() except FullTextError: @@ -155,12 +185,10 @@ def populate_fulltext_columns( def drop_fulltext_columns( engine: Engine, *, - db_schema: str | None = None, drop_indexes: bool = True, dry_run: bool = False, ) -> tuple[FullTextResult, ...]: """Remove tsvector sidecar columns and their associated GIN indexes.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "drop_fulltext_on_table", "Full-text search") targets = backend.fulltext_targets @@ -174,8 +202,8 @@ def drop_fulltext_columns( table_name=cfg.table_name, vector_column_name=cfg.vector_column_name, index_name=cfg.index_name, - db_schema=db_schema, drop_indexes=drop_indexes, + schema_tag=_schema_tag_for_target(cfg.table_name), ) backend.unregister_fulltext_metadata() except FullTextError: @@ -227,10 +255,10 @@ def install_fulltext_command( with console.status("Managing PostgreSQL full-text sidecar columns..."): results = install_fulltext_columns( engine, - db_schema=conn.db_schema, create_indexes=create_indexes, fastupdate=fastupdate, dry_run=dry_run, + resolved=conn.resolved, ) console.print(render_fulltext_results(results)) console.print(render_fulltext_summary(results, action="install", dry_run=dry_run)) @@ -251,7 +279,6 @@ def populate_fulltext_command( with console.status("Managing PostgreSQL full-text sidecar columns..."): results = populate_fulltext_columns( engine, - db_schema=conn.db_schema, regconfig=regconfig, dry_run=dry_run, ) @@ -275,7 +302,6 @@ def drop_fulltext_command( with console.status("Managing PostgreSQL full-text sidecar columns..."): results = drop_fulltext_columns( engine, - db_schema=conn.db_schema, drop_indexes=drop_indexes, dry_run=dry_run, ) diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index b639b5f..c3babcb 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from contextlib import ExitStack from dataclasses import dataclass from typing import Any, Mapping, Sequence @@ -10,14 +11,16 @@ from sqlalchemy.exc import DBAPIError, IntegrityError import typer +from oa_configurator import ResolvedCDMDatabase, ensure_schema, guard_schema_provenance_for, physical_schema_of, supports_schemas + from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY -from ..backends import Backend, resolve_backend, backend_supports -from ._cli_utils import ReservedSchema, Status, dry_label, dry_status, omop_command, reject_reserved_schema +from ..backends import resolve_backend, backend_supports +from ..config import MAINTENANCE_SCHEMA +from ._cli_utils import Status, dry_label, dry_status, omop_command from .tables import ( MaintenanceTable, TableCategory, - schema_adjusted_metadata, select_omop_tables, ) from .ui import ( @@ -34,6 +37,7 @@ class IndexTarget: table_name: str category: TableCategory + schema_tag: str index_name: str column_names: tuple[str, ...] unique: bool @@ -78,13 +82,41 @@ def _is_plain_index(reflected: Mapping[str, Any]) -> bool: if reflected.get("duplicates_constraint"): return False dialect_options = reflected.get("dialect_options") or {} - if dialect_options.get("postgresql_where"): + if _dialect_option(dialect_options, "where") is not None: return False - if dialect_options.get("postgresql_using"): + if _dialect_option(dialect_options, "using") is not None: return False return True +def _dialect_option(dialect_options: Mapping[str, Any], suffix: str) -> Any | None: + """Find a reflected index's dialect-specific option, regardless of dialect. + + SQLAlchemy always prefixes a reflected index's dialect-specific options + with the dialect name, e.g. ``"postgresql_where"`` or ``"sqlite_where"``. + Matching by suffix instead of a hardcoded dialect name means this works + for any dialect, current or future, with no per-dialect registration + needed. + + Parameters + ---------- + dialect_options : Mapping[str, Any] + A reflected index's ``dialect_options`` mapping. + suffix : str + The option name to look for, without its dialect prefix (e.g. + ``"where"``, ``"using"``). + + Returns + ------- + Any | None + The matching option's value, or None if no dialect set it. + """ + for key, value in dialect_options.items(): + if key.endswith(f"_{suffix}") and value: + return value + return None + + def _find_equivalent_index( existing_indexes: Sequence[Mapping[str, Any]], column_names: tuple[str, ...], @@ -182,10 +214,11 @@ def _describe_shape_conflict(reflected: Mapping[str, Any]) -> str: reasons: list[str] = [] if reflected.get("duplicates_constraint"): reasons.append("backs a UNIQUE/PRIMARY KEY constraint") - if dialect_options.get("postgresql_where"): + if _dialect_option(dialect_options, "where") is not None: reasons.append("has a partial WHERE predicate") - if dialect_options.get("postgresql_using"): - reasons.append(f"uses non-btree access method '{dialect_options['postgresql_using']}'") + using = _dialect_option(dialect_options, "using") + if using is not None: + reasons.append(f"uses non-btree access method '{using}'") if not reasons: reasons.append("has an unsupported definition") return ", ".join(reasons) @@ -219,23 +252,22 @@ def _schema_key(db_schema: str | None) -> str: return db_schema or "" -def get_bookkeeping_schema(backend: Backend) -> str | None: +def get_bookkeeping_schema(connection: sa.Connection) -> str | None: """Return the reserved schema name for the dropped-index bookkeeping table. Parameters ---------- - backend : Backend - The resolved database backend. + connection : sqlalchemy.Connection + The connection the bookkeeping table would be created on. Returns ------- str or None - ReservedSchema.MAINTENANCE.value on backends that override - Backend.ensure_schema() (i.e. support named schemas, like - PostgreSQL), or None on backends that don't (like SQLite). + MAINTENANCE_SCHEMA on a dialect with a genuine multi-schema concept + (like PostgreSQL), or None on one that doesn't (like SQLite). """ - if backend_supports(backend, "ensure_schema"): - return ReservedSchema.MAINTENANCE.value + if supports_schemas(connection): + return MAINTENANCE_SCHEMA return None @@ -284,7 +316,6 @@ def _dropped_indexes_table(bookkeeping_schema: str | None) -> sa.Table: def _record_captured_index( connection: sa.Connection, - backend: Backend, *, table_name: str, db_schema: str | None, @@ -307,8 +338,6 @@ def _record_captured_index( ---------- connection : sqlalchemy.Connection Open connection/transaction the capture is recorded on. - backend : Backend - The resolved database backend. table_name : str Name of the table the foreign index belongs to. db_schema : str or None @@ -327,8 +356,8 @@ def _record_captured_index( True if the capture was recorded, False if a pending capture already existed for this table/schema/column-set/uniqueness. """ - bookkeeping_schema = get_bookkeeping_schema(backend) - backend.ensure_schema(connection, bookkeeping_schema) + bookkeeping_schema = get_bookkeeping_schema(connection) + ensure_schema(connection, bookkeeping_schema) bookkeeping_table = _dropped_indexes_table(bookkeeping_schema) bookkeeping_table.create(bind=connection, checkfirst=True) @@ -360,7 +389,6 @@ def _record_captured_index( def _peek_captured_index( connection: sa.Connection, - backend: Backend, *, table_name: str, db_schema: str | None, @@ -377,8 +405,6 @@ def _peek_captured_index( ---------- connection : sqlalchemy.Connection Open connection the lookup is performed on. - backend : Backend - The resolved database backend. table_name : str Name of the table the index belongs to. db_schema : str or None @@ -400,7 +426,7 @@ def _peek_captured_index( The matched bookkeeping row, for use in a later delete by id. None if nothing is captured for this table/schema/column-set/uniqueness. """ - bookkeeping_schema = get_bookkeeping_schema(backend) + bookkeeping_schema = get_bookkeeping_schema(connection) inspector = sa.inspect(connection) if not inspector.has_table(_DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema): return None, None, None @@ -420,7 +446,6 @@ def _peek_captured_index( def _restore_captured_index( connection: sa.Connection, - backend: Backend, *, table_name: str, db_schema: str | None, @@ -435,8 +460,6 @@ def _restore_captured_index( ---------- connection : sqlalchemy.Connection Open connection/transaction the index is created on. - backend : Backend - The resolved database backend. table_name : str Name of the table to recreate the index on. db_schema : str or None @@ -465,7 +488,6 @@ def _restore_captured_index( """ restored_index_name, bookkeeping_table, row = _peek_captured_index( connection=connection, - backend=backend, table_name=table_name, db_schema=db_schema, column_names=column_names, @@ -473,10 +495,10 @@ def _restore_captured_index( ) if restored_index_name is None or bookkeeping_table is None or row is None: return None - + # A lightweight, untyped Table (no autoload_with reflection) is sufficient: # CREATE INDEX DDL only needs column names, not real types, PKs, FKs, or - # constraints -- reflecting the whole table would cost several extra + # constraints. Reflecting the whole table would cost several extra # catalog round-trips to fetch metadata this function never uses. lightweight_table = sa.Table( table_name, sa.MetaData(), @@ -501,22 +523,16 @@ def _restore_captured_index( def _schema_metadata_indexes( tables: list[MaintenanceTable], - db_schema: str | None, ) -> dict[tuple[str, str], sa.Index]: - """Return a (table_name, index_name) → Index mapping from ORM metadata, adjusted for db_schema if provided.""" - indexes: dict[tuple[str, str], sa.Index] = {} - - if db_schema is None: - for table in tables: - for index in table.table.indexes: - indexes[(table.table_name, str(index.name))] = index - return indexes - - _, copied_tables = schema_adjusted_metadata(tables, db_schema=db_schema) - for table_name, table in copied_tables.items(): - for index in table.indexes: - indexes[(table_name, str(index.name))] = index + """Return a (table_name, index_name) -> Index mapping from ORM metadata. + Index name/columns don't depend on which schema a table is tagged with, + so this reads straight off each table's own ORM metadata. + """ + indexes: dict[tuple[str, str], sa.Index] = {} + for table in tables: + for index in table.table.indexes: + indexes[(table.table_name, str(index.name))] = index return indexes @@ -600,7 +616,6 @@ def _resolve_physical_cluster_name( def collect_index_targets( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, ) -> list[IndexTarget]: """List ORM-defined indexes that currently exist in the target database.""" @@ -609,10 +624,11 @@ def collect_index_targets( targets: list[IndexTarget] = [] for table in selected_tables: - if not inspector.has_table(table.table_name, schema=db_schema): + table_schema = physical_schema_of(engine, schema_tag=table.schema_tag) + if not inspector.has_table(table.table_name, schema=table_schema): continue - existing_indexes = inspector.get_indexes(table.table_name, schema=db_schema) + existing_indexes = inspector.get_indexes(table.table_name, schema=table_schema) existing_index_names = {index["name"] for index in existing_indexes} for metadata_index in sorted(table.table.indexes, key=lambda idx: idx.name or ""): @@ -630,6 +646,7 @@ def collect_index_targets( IndexTarget( table_name=table.table_name, category=table.category, + schema_tag=table.schema_tag, index_name=physical_name, column_names=column_names, unique=unique, @@ -668,285 +685,307 @@ def manage_indexes( engine: sa.Engine, *, enable: bool, - db_schema: str | None = None, vocabulary_included: bool = False, dry_run: bool = False, cluster: bool = True, + resolved: ResolvedCDMDatabase | None = None, ) -> list[IndexManagementResult]: """Create or drop all ORM-defined indexes. CLUSTERs tables when enabling and cluster=True.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) inspector = sa.inspect(engine) selected_tables = select_omop_tables(vocabulary_included=vocabulary_included) - metadata_indexes = _schema_metadata_indexes(selected_tables, db_schema) + metadata_indexes = _schema_metadata_indexes(selected_tables) clustering_supported = backend_supports(backend, "cluster_table") results: list[IndexManagementResult] = [] - for table in selected_tables: - if not inspector.has_table(table.table_name, schema=db_schema): - continue - - existing_indexes = inspector.get_indexes(table.table_name, schema=db_schema) - existing_index_names = {index["name"] for index in existing_indexes} + with ExitStack() as guard_stack: + if not dry_run: + tables_by_schema_tag: dict[str, list[sa.Table]] = {} + for table in selected_tables: + tables_by_schema_tag.setdefault(table.schema_tag, []).append(table.table) + guard_connection = guard_stack.enter_context(engine.begin()) + # One provenance guard per schema_tag (count only known at runtime); ExitStack defers every write until the block below succeeds. + for schema_tag, tables in tables_by_schema_tag.items(): + # A same-named index could already exist under a drifted schema, attached to an unrelated table. + guard_stack.enter_context( + guard_schema_provenance_for(guard_connection, resolved, schema_tag=schema_tag, tables=tables) + ) - created_any = False - clustered_now = False - physical_index_names: dict[str, str] = {} + for table in selected_tables: + db_schema = physical_schema_of(engine, schema_tag=table.schema_tag) + if not inspector.has_table(table.table_name, schema=db_schema): + continue - for metadata_index in sorted(table.table.indexes, key=lambda idx: idx.name or ""): - index_name = str(metadata_index.name) - column_names = tuple(column.name for column in metadata_index.columns) - unique = bool(metadata_index.unique) - exists = index_name in existing_index_names - should_apply = ( - not enable - ) or ( - enable and not exists - ) + existing_indexes = inspector.get_indexes(table.table_name, schema=db_schema) + existing_index_names = {index["name"] for index in existing_indexes} + + created_any = False + clustered_now = False + physical_index_names: dict[str, str] = {} + + for metadata_index in sorted(table.table.indexes, key=lambda idx: idx.name or ""): + index_name = str(metadata_index.name) + column_names = tuple(column.name for column in metadata_index.columns) + unique = bool(metadata_index.unique) + exists = index_name in existing_index_names + should_apply = ( + not enable + ) or ( + enable and not exists + ) - if not should_apply: - physical_index_names[index_name] = index_name - continue + if not should_apply: + physical_index_names[index_name] = index_name + continue - schema_index = metadata_indexes[(table.table_name, index_name)] - # Plain create/drop succeeding is the common case for both live and - # dry runs, so it's the default outcome; every branch below only - # overrides it for a foreign-index or already-in-place case. - outcome = _IndexOutcome( - status=dry_status(dry_run), - detail=dry_label( - dry_run, - planned="metadata-defined index would be dropped" if not enable else "metadata-defined index would be created", - applied="metadata-defined index dropped" if not enable else "metadata-defined index created", - ), - physical_name=index_name, - ) + schema_index = metadata_indexes[(table.table_name, index_name)] + # Plain create/drop succeeding is the common case for both live and + # dry runs, so it's the default outcome; every branch below only + # overrides it for a foreign-index or already-in-place case. + outcome = _IndexOutcome( + status=dry_status(dry_run), + detail=dry_label( + dry_run, + planned="metadata-defined index would be dropped" if not enable else "metadata-defined index would be created", + applied="metadata-defined index dropped" if not enable else "metadata-defined index created", + ), + physical_name=index_name, + ) - # Each index gets its own connection: a transaction when actually - # mutating (not dry_run, so WAL is committed and checkpointable - # before the next index build begins), a plain read-only - # connection when only previewing. - connection_factory = engine.begin if not dry_run else engine.connect - with connection_factory() as connection: - if not enable: - if not dry_run: - existed_before_drop = backend.index_exists(connection, index_name, db_schema) - else: - existed_before_drop = exists - if not existed_before_drop: - # Index under a different naming scheme than ours - equivalent_name = _find_equivalent_index(existing_indexes, column_names, unique) - if equivalent_name is not None: - if not dry_run: - captured = _record_captured_index( - connection, backend, - table_name=table.table_name, db_schema=db_schema, - index_name=equivalent_name, - column_names=column_names, unique=unique, - ) - else: - pending_capture, _, _ = _peek_captured_index( - connection, backend, - table_name=table.table_name, db_schema=db_schema, - column_names=column_names, unique=unique, - ) - captured = pending_capture is None - if captured: + # Each index gets its own connection: a transaction when actually + # mutating (not dry_run, so WAL is committed and checkpointable + # before the next index build begins), a plain read-only + # connection when only previewing. + connection_factory = engine.begin if not dry_run else engine.connect + with connection_factory() as connection: + if not enable: + if not dry_run: + existed_before_drop = backend.index_exists( + connection, index_name, schema_tag=table.schema_tag + ) + else: + existed_before_drop = exists + if not existed_before_drop: + # Index under a different naming scheme than ours + equivalent_name = _find_equivalent_index(existing_indexes, column_names, unique) + if equivalent_name is not None: if not dry_run: - backend.drop_index_if_exists(connection, equivalent_name, db_schema) - outcome = _IndexOutcome( - status=dry_status(dry_run, Status.CAPTURED), - detail=dry_label( - dry_run, - planned=f"foreign index '{equivalent_name}' would be captured and dropped for bulk load", - applied=f"foreign index '{equivalent_name}' captured and dropped for bulk load", - ), - physical_name=equivalent_name, - ) - else: - # A different foreign index for this table/column-set is - # already captured and awaiting restore. Leave this one - # in place rather than dropping something we can no - # longer track. - outcome = _IndexOutcome( - status=Status.WARNING, - detail=dry_label( - dry_run, - planned=( - f"foreign index '{equivalent_name}' would be left in place: a different " - "foreign index for this table/column-set is already captured and " - "awaiting restore" + captured = _record_captured_index( + connection, + table_name=table.table_name, db_schema=db_schema, + index_name=equivalent_name, + column_names=column_names, unique=unique, + ) + else: + pending_capture, _, _ = _peek_captured_index( + connection, + table_name=table.table_name, db_schema=db_schema, + column_names=column_names, unique=unique, + ) + captured = pending_capture is None + if captured: + if not dry_run: + backend.drop_index_if_exists( + connection, equivalent_name, schema_tag=table.schema_tag + ) + outcome = _IndexOutcome( + status=dry_status(dry_run, Status.CAPTURED), + detail=dry_label( + dry_run, + planned=f"foreign index '{equivalent_name}' would be captured and dropped for bulk load", + applied=f"foreign index '{equivalent_name}' captured and dropped for bulk load", ), - applied=( - f"foreign index '{equivalent_name}' left in place: a different " - "foreign index for this table/column-set is already captured and " - "awaiting restore" + physical_name=equivalent_name, + ) + else: + # A different foreign index for this table/column-set is + # already captured and awaiting restore. Leave this one + # in place rather than dropping something we can no + # longer track. + outcome = _IndexOutcome( + status=Status.WARNING, + detail=dry_label( + dry_run, + planned=( + f"foreign index '{equivalent_name}' would be left in place: a different " + "foreign index for this table/column-set is already captured and " + "awaiting restore" + ), + applied=( + f"foreign index '{equivalent_name}' left in place: a different " + "foreign index for this table/column-set is already captured and " + "awaiting restore" + ), ), - ), - physical_name=equivalent_name, - ) - else: - conflict = _find_shape_conflict(existing_indexes, column_names, unique) - if conflict is not None: - conflict_name = str(conflict["name"]) - outcome = _IndexOutcome( - status=Status.WARNING, - detail=dry_label( - dry_run, - planned=f"foreign index '{conflict_name}' {_describe_shape_conflict(conflict)}; would be left in place", - applied=f"foreign index '{conflict_name}' {_describe_shape_conflict(conflict)}; left in place", - ), - physical_name=conflict_name, - ) + physical_name=equivalent_name, + ) else: - outcome = _IndexOutcome( - status=Status.SKIPPED, - detail="metadata-defined index already absent (skipped)", - physical_name=index_name, - ) - elif not dry_run: - backend.drop_index_if_exists(connection, index_name, db_schema) - # outcome stays default: applied / "metadata-defined index dropped" - # dry-run, existed_before_drop True: outcome stays default ("would be dropped") - else: - if not dry_run: - restored_name = _restore_captured_index( - connection, backend, - table_name=table.table_name, db_schema=db_schema, - column_names=column_names, unique=unique, - ) + conflict = _find_shape_conflict(existing_indexes, column_names, unique) + if conflict is not None: + conflict_name = str(conflict["name"]) + outcome = _IndexOutcome( + status=Status.WARNING, + detail=dry_label( + dry_run, + planned=f"foreign index '{conflict_name}' {_describe_shape_conflict(conflict)}; would be left in place", + applied=f"foreign index '{conflict_name}' {_describe_shape_conflict(conflict)}; left in place", + ), + physical_name=conflict_name, + ) + else: + outcome = _IndexOutcome( + status=Status.SKIPPED, + detail="metadata-defined index already absent (skipped)", + physical_name=index_name, + ) + elif not dry_run: + backend.drop_index_if_exists(connection, index_name, schema_tag=table.schema_tag) + # outcome stays default: applied / "metadata-defined index dropped" + # dry-run, existed_before_drop True: outcome stays default ("would be dropped") else: - restored_name, _, _ = _peek_captured_index( - connection, backend, - table_name=table.table_name, db_schema=db_schema, - column_names=column_names, unique=unique, - ) - if restored_name is not None: if not dry_run: - created_any = True - outcome = _IndexOutcome( - status=dry_status(dry_run, Status.RESTORED), - detail=dry_label( - dry_run, - planned=f"foreign index '{restored_name}' would be restored from bulk-load capture", - applied=f"foreign index '{restored_name}' restored from bulk-load capture", - ), - physical_name=restored_name, - ) - else: - equivalent_name = _find_equivalent_index(existing_indexes, column_names, unique) - if equivalent_name is not None: + restored_name = _restore_captured_index( + connection, + table_name=table.table_name, db_schema=db_schema, + column_names=column_names, unique=unique, + ) + else: + restored_name, _, _ = _peek_captured_index( + connection, + table_name=table.table_name, db_schema=db_schema, + column_names=column_names, unique=unique, + ) + if restored_name is not None: + if not dry_run: + created_any = True outcome = _IndexOutcome( - status=Status.SKIPPED, + status=dry_status(dry_run, Status.RESTORED), detail=dry_label( dry_run, - planned=f"equivalent foreign index '{equivalent_name}' already provides this coverage (would skip creation)", - applied=f"equivalent foreign index '{equivalent_name}' already provides this coverage (skipped)", + planned=f"foreign index '{restored_name}' would be restored from bulk-load capture", + applied=f"foreign index '{restored_name}' restored from bulk-load capture", ), - physical_name=equivalent_name, + physical_name=restored_name, ) - elif not dry_run: - savepoint = connection.begin_nested() - try: - schema_index.create(bind=connection, checkfirst=True) - except DBAPIError as exc: - savepoint.rollback() - if "already exists" not in str(exc.orig).lower(): - raise + else: + equivalent_name = _find_equivalent_index(existing_indexes, column_names, unique) + if equivalent_name is not None: outcome = _IndexOutcome( status=Status.SKIPPED, - detail="metadata-defined index already exists (skipped)", - physical_name=index_name, + detail=dry_label( + dry_run, + planned=f"equivalent foreign index '{equivalent_name}' already provides this coverage (would skip creation)", + applied=f"equivalent foreign index '{equivalent_name}' already provides this coverage (skipped)", + ), + physical_name=equivalent_name, ) - else: - savepoint.commit() - created_any = True - # outcome stays default: applied / "metadata-defined index created" - # dry-run, no restore, no equivalent: outcome stays default ("would be created") - - physical_name = outcome.physical_name - physical_index_names[index_name] = physical_name - results.append( - IndexManagementResult( - operation="index", - table_name=table.table_name, - category=table.category, - index_name=physical_name, - column_names=column_names, - unique=unique, - clustered=metadata_index.info.get(OMOP_CLUSTER_INDEX_INFO_KEY) is True, - enable=enable, - status=outcome.status, - detail=outcome.detail, + elif not dry_run: + savepoint = connection.begin_nested() + try: + schema_index.create(bind=connection, checkfirst=True) + except DBAPIError as exc: + savepoint.rollback() + if "already exists" not in str(exc.orig).lower(): + raise + outcome = _IndexOutcome( + status=Status.SKIPPED, + detail="metadata-defined index already exists (skipped)", + physical_name=index_name, + ) + else: + savepoint.commit() + created_any = True + # outcome stays default: applied / "metadata-defined index created" + # dry-run, no restore, no equivalent: outcome stays default ("would be created") + + physical_name = outcome.physical_name + physical_index_names[index_name] = physical_name + results.append( + IndexManagementResult( + operation="index", + table_name=table.table_name, + category=table.category, + schema_tag=table.schema_tag, + index_name=physical_name, + column_names=column_names, + unique=unique, + clustered=metadata_index.info.get(OMOP_CLUSTER_INDEX_INFO_KEY) is True, + enable=enable, + status=outcome.status, + detail=outcome.detail, + ) ) - ) - # Clustering for perfomance is a separate operation from index creation - if enable: - cluster_index_name = _cluster_target_name(table) - if cluster_index_name is not None: - cluster_columns = _cluster_column_names(table, cluster_index_name) - if cluster_index_name in physical_index_names: - # Resolved authoritatively from what actually happened in this - # run's per-index loop (own name, captured, restored, or a - # skip-equivalent) -- more precise than re-deriving from the - # now-stale existing_indexes snapshot, since e.g. a - # just-restored index wouldn't appear in it. - physical_cluster_name = physical_index_names[cluster_index_name] - else: - # Primary-key-based cluster target: never entered the per-index - # loop, so resolve it the same way the standalone `indexes - # cluster` command does. - physical_cluster_name = _resolve_physical_cluster_name( - existing_indexes, - cluster_index_name, - cluster_columns, - ) - if not clustering_supported or not cluster: - results.append( - IndexManagementResult( - operation="cluster", - table_name=table.table_name, - category=table.category, - index_name=physical_cluster_name, - column_names=cluster_columns, - unique=False, - clustered=True, - enable=enable, - status=Status.SKIPPED, - detail=( - f"cluster metadata present but unsupported on {backend.name}" - if not clustering_supported - else "clustering skipped (run 'indexes cluster' to apply)" - ), + # Clustering for perfomance is a separate operation from index creation + if enable: + cluster_index_name = _cluster_target_name(table) + if cluster_index_name is not None: + cluster_columns = _cluster_column_names(table, cluster_index_name) + if cluster_index_name in physical_index_names: + # Resolved authoritatively from what actually happened in this + # run's per-index loop (own name, captured, restored, or a + # skip-equivalent) -- more precise than re-deriving from the + # now-stale existing_indexes snapshot, since e.g. a + # just-restored index wouldn't appear in it. + physical_cluster_name = physical_index_names[cluster_index_name] + else: + # Primary-key-based cluster target: never entered the per-index + # loop, so resolve it the same way the standalone `indexes + # cluster` command does. + physical_cluster_name = _resolve_physical_cluster_name( + existing_indexes, + cluster_index_name, + cluster_columns, ) - ) - else: - if not dry_run: - with engine.begin() as connection: - backend.cluster_table(connection, table.table_name, physical_cluster_name, db_schema) - clustered_now = True - - results.append( - IndexManagementResult( - operation="cluster", - table_name=table.table_name, - category=table.category, - index_name=physical_cluster_name, - column_names=cluster_columns, - unique=False, - clustered=True, - enable=enable, - status=dry_status(dry_run), - detail=dry_label(dry_run, "table would be clustered using ORM-defined metadata", "table clustered using ORM-defined metadata"), + if not clustering_supported or not cluster: + results.append( + IndexManagementResult( + operation="cluster", + table_name=table.table_name, + category=table.category, + schema_tag=table.schema_tag, + index_name=physical_cluster_name, + column_names=cluster_columns, + unique=False, + clustered=True, + enable=enable, + status=Status.SKIPPED, + detail=( + f"cluster metadata present but unsupported on {backend.name}" + if not clustering_supported + else "clustering skipped (run 'indexes cluster' to apply)" + ), + ) + ) + else: + if not dry_run: + with engine.begin() as connection: + backend.cluster_table( + connection, table.table_name, physical_cluster_name, schema_tag=table.schema_tag + ) + clustered_now = True + + results.append( + IndexManagementResult( + operation="cluster", + table_name=table.table_name, + category=table.category, + schema_tag=table.schema_tag, + index_name=physical_cluster_name, + column_names=cluster_columns, + unique=False, + clustered=True, + enable=enable, + status=dry_status(dry_run), + detail=dry_label(dry_run, "table would be clustered using ORM-defined metadata", "table clustered using ORM-defined metadata"), + ) ) - ) - if not dry_run and (created_any or clustered_now): - with engine.connect() as connection: - backend.analyze_table(connection, table.table_name, db_schema) - connection.commit() + if not dry_run and (created_any or clustered_now): + with engine.connect() as connection: + backend.analyze_table(connection, table.table_name, schema_tag=table.schema_tag) + connection.commit() return results @@ -974,9 +1013,9 @@ def disable_indexes_command( results = manage_indexes( engine, enable=False, - db_schema=conn.db_schema, vocabulary_included=vocabulary_included, dry_run=dry_run, + resolved=conn.resolved, ) console.print(render_index_results(results)) console.print(render_index_summary(results, dry_run=dry_run)) @@ -1010,10 +1049,10 @@ def enable_indexes_command( results = manage_indexes( engine, enable=True, - db_schema=conn.db_schema, vocabulary_included=vocabulary_included, dry_run=dry_run, cluster=cluster, + resolved=conn.resolved, ) console.print(render_index_results(results)) console.print(render_index_summary(results, dry_run=dry_run)) @@ -1035,7 +1074,7 @@ def cluster_tables_command( """CLUSTER tables using their ORM-designated cluster index. Physically rewrites table data sorted by the cluster index for improved sequential-scan - performance. Requires approximately 2× the table size in free disk space per table. + performance. Requires approximately 2x the table size in free disk space per table. Run this after 'indexes enable' once you have confirmed sufficient disk headroom. On Docker, check Docker Desktop → Resources → Virtual Disk Limit before running on @@ -1050,43 +1089,60 @@ def cluster_tables_command( selected_tables = select_omop_tables(vocabulary_included=vocabulary_included) results: list[IndexManagementResult] = [] - for table in selected_tables: - if not inspector.has_table(table.table_name, schema=conn.db_schema): - continue + with ExitStack() as guard_stack: + if not dry_run: + tables_by_schema_tag: dict[str, list[sa.Table]] = {} + for table in selected_tables: + tables_by_schema_tag.setdefault(table.schema_tag, []).append(table.table) + guard_connection = guard_stack.enter_context(engine.begin()) + # One provenance guard per schema_tag (count only known at runtime); ExitStack defers every write until the block below succeeds. + for schema_tag, tables in tables_by_schema_tag.items(): + # CLUSTER physically rewrites the table's heap: guard against a drifted schema the same as any other DDL. + guard_stack.enter_context( + guard_schema_provenance_for(guard_connection, conn.resolved, schema_tag=schema_tag, tables=tables) + ) - cluster_index_name = _cluster_target_name(table) - if cluster_index_name is None: - continue + for table in selected_tables: + table_schema = physical_schema_of(engine, schema_tag=table.schema_tag) + if not inspector.has_table(table.table_name, schema=table_schema): + continue - cluster_columns = _cluster_column_names(table, cluster_index_name) - existing_indexes = inspector.get_indexes(table.table_name, schema=conn.db_schema) - physical_cluster_name = _resolve_physical_cluster_name( - existing_indexes, - cluster_index_name, - cluster_columns, - ) + cluster_index_name = _cluster_target_name(table) + if cluster_index_name is None: + continue - if not dry_run: - with engine.begin() as connection: - backend.cluster_table(connection, table.table_name, physical_cluster_name, conn.db_schema) - with engine.connect() as connection: - backend.analyze_table(connection, table.table_name, conn.db_schema) - connection.commit() - - results.append( - IndexManagementResult( - operation="cluster", - table_name=table.table_name, - category=table.category, - index_name=physical_cluster_name, - column_names=cluster_columns, - unique=False, - clustered=True, - enable=True, - status=dry_status(dry_run), - detail=dry_label(dry_run, "table would be clustered and analyzed", "table clustered and analyzed"), + cluster_columns = _cluster_column_names(table, cluster_index_name) + existing_indexes = inspector.get_indexes(table.table_name, schema=table_schema) + physical_cluster_name = _resolve_physical_cluster_name( + existing_indexes, + cluster_index_name, + cluster_columns, + ) + + if not dry_run: + with engine.begin() as connection: + backend.cluster_table( + connection, table.table_name, physical_cluster_name, schema_tag=table.schema_tag + ) + with engine.connect() as connection: + backend.analyze_table(connection, table.table_name, schema_tag=table.schema_tag) + connection.commit() + + results.append( + IndexManagementResult( + operation="cluster", + table_name=table.table_name, + category=table.category, + schema_tag=table.schema_tag, + index_name=physical_cluster_name, + column_names=cluster_columns, + unique=False, + clustered=True, + enable=True, + status=dry_status(dry_run), + detail=dry_label(dry_run, "table would be clustered and analyzed", "table clustered and analyzed"), + ) ) - ) console.print(render_index_results(results)) console.print(render_index_summary(results, dry_run=dry_run)) diff --git a/omop_alchemy/maintenance/cli_schema.py b/omop_alchemy/maintenance/cli_schema.py index b82e2dc..88f4d09 100644 --- a/omop_alchemy/maintenance/cli_schema.py +++ b/omop_alchemy/maintenance/cli_schema.py @@ -1,4 +1,4 @@ -"""Schema subapp: thin shim re-exporting all domain types and wiring five CLI commands.""" +"""Schema subapp: thin shim re-exporting all domain types and wiring seven CLI commands.""" from __future__ import annotations @@ -101,7 +101,8 @@ def doctor_command( with console.status("Running maintenance doctor checks..."): report = collect_doctor_report( engine=engine, - db_schema=conn.db_schema, + resolved=conn.resolved, + db_schema=conn.resolved.schema_name, resource_name=conn.resource_name, vocabulary_included=vocabulary_included, deep=deep, @@ -128,7 +129,7 @@ def reconcile_schema_command( ) -> None: """Compare ORM-managed SQLAlchemy metadata against the current target database schema.""" with console.status("Reconciling ORM metadata against target database schema..."): - report = reconcile_schema(engine, db_schema=conn.db_schema, vocabulary_included=vocabulary_included) + report = reconcile_schema(engine, resolved=conn.resolved, vocabulary_included=vocabulary_included) console.print(render_reconciliation_results(report.table_results)) console.print(render_reconciliation_issues(report.issues)) console.print(render_reconciliation_summary(report)) @@ -147,13 +148,19 @@ def create_missing_tables_command( dry_run: bool = False, ) -> None: """Create missing ORM-managed OMOP tables from metadata.""" - with console.status("Creating missing tables..."): - results = create_missing_tables( - engine, - db_schema=conn.db_schema, - vocabulary_included=vocabulary_included, - dry_run=dry_run, - ) + vocab_engine = conn.resolved.vocab_engine_for(engine) + try: + with console.status("Creating missing tables..."): + results = create_missing_tables( + engine, + vocab_engine=vocab_engine, + vocabulary_included=vocabulary_included, + dry_run=dry_run, + resolved=conn.resolved, + ) + finally: + if vocab_engine is not engine: + vocab_engine.dispose() console.print(render_table_creation_results(results)) console.print(render_table_creation_summary(results, dry_run=dry_run)) @@ -178,9 +185,10 @@ def data_summary_command( with console.status("Collecting table summary..."): results = collect_data_summary( engine, - db_schema=conn.db_schema, vocabulary_included=vocabulary_included, existing_only=not include_missing, ) console.print(render_data_summary_results(results)) console.print(render_data_summary_summary(results)) + + diff --git a/omop_alchemy/maintenance/cli_schema_doctor.py b/omop_alchemy/maintenance/cli_schema_doctor.py index 6fe5ad3..eb60395 100644 --- a/omop_alchemy/maintenance/cli_schema_doctor.py +++ b/omop_alchemy/maintenance/cli_schema_doctor.py @@ -5,9 +5,9 @@ from dataclasses import dataclass import sqlalchemy as sa +from oa_configurator import Dialect, ResolvedDatabase -from omop_alchemy.backends.resolve import SupportedDialect - +from ..backends import backend_supports, resolve_backend from ._cli_utils import Status from .cli_foreign_keys import ( ForeignKeyStatusResult, @@ -101,6 +101,18 @@ def _build_recommendations( action="Review `omop-alchemy reconcile-schema` output before continuing with ETL or maintenance work.", ) ) + if any(issue.status == Status.RELOCATED for issue in reconciliation.issues): + recommendations.append( + DoctorRecommendation( + status=Status.WARNING, + summary="Some tables were found under a different schema than expected.", + action=( + "Run `omop-config acknowledge-schema-migration` if this was a " + "deliberate change, or `omop-config drop-orphan-schema-tables` to " + "clean up an orphaned copy." + ), + ) + ) if foreign_key_status is not None and any( item.disabled_trigger_count > 0 for item in foreign_key_status @@ -128,7 +140,7 @@ def _build_recommendations( ) ) - if info.backend == SupportedDialect.POSTGRESQL and info.pg_dump_path is None: + if info.backend == Dialect.POSTGRESQL and info.pg_dump_path is None: recommendations.append( DoctorRecommendation( status=Status.WARNING, @@ -138,7 +150,7 @@ def _build_recommendations( ) if ( - info.backend == SupportedDialect.POSTGRESQL + info.backend == Dialect.POSTGRESQL and info.pg_restore_path is None and info.psql_path is None ): @@ -165,6 +177,7 @@ def _build_recommendations( def collect_doctor_report( *, engine: sa.engine.Engine, + resolved: ResolvedDatabase | None = None, db_schema: str | None = None, resource_name: str | None = None, vocabulary_included: bool = True, @@ -178,6 +191,9 @@ def collect_doctor_report( Already-resolved CDM engine (e.g. from the ``@omop_command`` decorator), reused for all database checks instead of re-resolving config. The caller retains ownership; this function does not dispose it. + resolved : ResolvedDatabase, optional + Forwarded to reconcile_schema (--deep only) so vocab/results tables + are compared against their own schema, not db_schema uniformly. db_schema : str, optional CDM schema associated with ``engine``. Omit to use its default schema. resource_name : str, optional @@ -225,6 +241,7 @@ def collect_doctor_report( if deep: reconciliation = reconcile_schema( engine, + resolved=resolved, db_schema=db_schema, vocabulary_included=vocabulary_included, ) @@ -255,11 +272,11 @@ def collect_doctor_report( ) ) - if info.backend == SupportedDialect.POSTGRESQL: + backend = resolve_backend(engine) + if backend_supports(backend, "get_fk_trigger_counts"): foreign_key_status = tuple( collect_foreign_key_trigger_status( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) ) @@ -280,10 +297,9 @@ def collect_doctor_report( ) ) - if deep: + if deep and backend_supports(backend, "count_fk_violations"): foreign_key_validation = validate_foreign_key_constraints( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) violating_tables = sum( @@ -305,6 +321,14 @@ def collect_doctor_report( ), ) ) + elif deep: + checks.append( + DoctorCheck( + name="foreign key validation", + status=Status.SKIPPED, + detail="Foreign key validation isn't supported on this backend.", + ) + ) else: checks.append( DoctorCheck( @@ -318,14 +342,14 @@ def collect_doctor_report( DoctorCheck( name="foreign keys", status=Status.SKIPPED, - detail="Foreign key trigger inspection is only available on PostgreSQL.", + detail="Foreign key trigger inspection isn't supported on this backend.", ) ) checks.append( DoctorCheck( name="foreign key validation", status=Status.SKIPPED, - detail="Foreign key validation is only available on PostgreSQL.", + detail="Foreign key validation isn't supported on this backend.", ) ) else: @@ -354,7 +378,7 @@ def collect_doctor_report( ) ) - if info.backend == SupportedDialect.POSTGRESQL: + if info.backend == Dialect.POSTGRESQL: backup_tools_ready = info.pg_dump_path is not None and ( info.pg_restore_path is not None or info.psql_path is not None ) diff --git a/omop_alchemy/maintenance/cli_schema_info.py b/omop_alchemy/maintenance/cli_schema_info.py index 09cbe0a..5f8c52a 100644 --- a/omop_alchemy/maintenance/cli_schema_info.py +++ b/omop_alchemy/maintenance/cli_schema_info.py @@ -10,11 +10,11 @@ import sqlalchemy as sa from sqlalchemy.exc import ArgumentError, SQLAlchemyError -from oa_configurator import ResolvedCDMDatabase, Resolver, load_stack_config +from oa_configurator import Dialect, ResolvedCDMDatabase, Resolver, load_stack_config from oa_configurator.loader import DEFAULT_CONFIG_PATH -from omop_alchemy.backends.resolve import SupportedDialect from omop_alchemy.config import OmopAlchemyConfig +from ..backends.resolve import backend_label from ._cli_utils import Status from .cli_schema_tables import collect_missing_tables from .tables import ( @@ -23,14 +23,6 @@ ) -def _backend_label(dialect_name: str) -> str: - from ..backends.resolve import _DIALECT_TO_BACKEND_MAP, SupportedDialect - try: - return _DIALECT_TO_BACKEND_MAP[SupportedDialect(dialect_name)].name - except (ValueError, KeyError): - return dialect_name - - # --------------------------------------------------------------------------- # info # --------------------------------------------------------------------------- @@ -146,7 +138,7 @@ def _command_support_for_backend( psql_path: str | None, ) -> tuple[CommandSupport, ...]: """Compute the readiness status of every CLI command given the current backend, connection state, and tool availability.""" - current_backend = _backend_label(backend) + current_backend = backend_label(backend) if not engine_created: blocked_detail = ( f"Backend resolved to {current_backend}, but the engine could not be created: {engine_error}" @@ -164,7 +156,7 @@ def _command_support_for_backend( f"Ready on {current_backend}." if connection_ready else blocked_detail ) - if backend == SupportedDialect.POSTGRESQL: + if backend == Dialect.POSTGRESQL: analyze_status = portable_status analyze_detail = ( "Ready on PostgreSQL; ANALYZE and VACUUM ANALYZE are both supported." @@ -185,7 +177,7 @@ def _command_support_for_backend( if connection_ready else blocked_detail ) - elif backend == "sqlite": + elif backend == Dialect.SQLITE: analyze_status = Status.LIMITED if connection_ready else Status.BLOCKED analyze_detail = ( "Ready on SQLite; ANALYZE is supported, but `--vacuum` is unavailable." @@ -250,18 +242,18 @@ def _command_support_for_backend( "PostgreSQL + pg_dump", ( Status.READY - if connection_ready and backend == SupportedDialect.POSTGRESQL and pg_dump_path is not None + if connection_ready and backend == Dialect.POSTGRESQL and pg_dump_path is not None else Status.BLOCKED - if backend == SupportedDialect.POSTGRESQL + if backend == Dialect.POSTGRESQL else Status.UNSUPPORTED if connection_ready else Status.BLOCKED ), ( "Ready on PostgreSQL; `pg_dump` is available." - if connection_ready and backend == SupportedDialect.POSTGRESQL and pg_dump_path is not None + if connection_ready and backend == Dialect.POSTGRESQL and pg_dump_path is not None else "PostgreSQL is configured, but `pg_dump` is not on PATH." - if connection_ready and backend == SupportedDialect.POSTGRESQL + if connection_ready and backend == Dialect.POSTGRESQL else f"Requires PostgreSQL. Current backend: {current_backend}." if connection_ready else blocked_detail @@ -272,18 +264,18 @@ def _command_support_for_backend( "PostgreSQL + pg_restore/psql", ( Status.READY - if connection_ready and backend == SupportedDialect.POSTGRESQL and (pg_restore_path is not None or psql_path is not None) + if connection_ready and backend == Dialect.POSTGRESQL and (pg_restore_path is not None or psql_path is not None) else Status.BLOCKED - if backend == SupportedDialect.POSTGRESQL + if backend == Dialect.POSTGRESQL else Status.UNSUPPORTED if connection_ready else Status.BLOCKED ), ( "Ready on PostgreSQL; restore client tooling is available." - if connection_ready and backend == SupportedDialect.POSTGRESQL and (pg_restore_path is not None or psql_path is not None) + if connection_ready and backend == Dialect.POSTGRESQL and (pg_restore_path is not None or psql_path is not None) else "PostgreSQL is configured, but neither `pg_restore` nor `psql` is on PATH." - if connection_ready and backend == SupportedDialect.POSTGRESQL + if connection_ready and backend == Dialect.POSTGRESQL else f"Requires PostgreSQL. Current backend: {current_backend}." if connection_ready else blocked_detail @@ -382,7 +374,6 @@ def collect_maintenance_info( connection_ready = True missing_tables = collect_missing_tables( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) missing_table_count = len(missing_tables) diff --git a/omop_alchemy/maintenance/cli_schema_reconcile.py b/omop_alchemy/maintenance/cli_schema_reconcile.py index 1fa5857..d34ef47 100644 --- a/omop_alchemy/maintenance/cli_schema_reconcile.py +++ b/omop_alchemy/maintenance/cli_schema_reconcile.py @@ -5,9 +5,16 @@ from dataclasses import dataclass import sqlalchemy as sa +from oa_configurator import ( + ResolvedDatabase, + find_table_in_other_schemas, + physical_schema_of, + supports_schemas, + validate_schema_tag, +) from sqlalchemy.engine.interfaces import ReflectedForeignKeyConstraint, ReflectedIndex -from ..backends import backend_supports, resolve_backend +from ..backends import Backend, backend_supports, resolve_backend from ._cli_utils import Severity, Status from .cli_indexes import _cluster_column_names, _cluster_target_name, _find_equivalent_index from .tables import ( @@ -66,19 +73,51 @@ class SchemaReconciliationReport: issues: tuple[ReconciliationIssue, ...] -def _schema_table(table: sa.Table, db_schema: str | None) -> sa.Table: - """Return table unchanged when db_schema is None, or a schema-qualified copy when a schema is specified.""" - if db_schema is None: - return table +def _effective_schema( + engine: sa.Engine, + resolved: ResolvedDatabase | None, + schema_tag: str | None, + db_schema: str | None, +) -> str | None: + """physical_schema_of(engine, schema_tag=schema_tag) when resolved is given, else db_schema. + + Uses physical_schema_of against engine to accommodate bare schema_tags. + """ + if resolved is None: + return db_schema + return physical_schema_of(engine, schema_tag=schema_tag) + + +def _schema_qualified_tables( + engine: sa.Engine, resolved: ResolvedDatabase | None, db_schema: str | None +) -> dict[int, sa.Table]: + """Schema-qualified copy of every table in Base.metadata, keyed by id() of the original. + Copied into one shared MetaData() since to_metadata() won't bring a referenced table's copy along on its own, which FK targets need present. + """ + from orm_loader.helpers import Base + + if resolved is None and db_schema is None: + return {id(table): table for table in Base.metadata.tables.values()} metadata = sa.MetaData() - return table.to_metadata( - metadata, - schema=db_schema, - referred_schema_fn=( - lambda _table, to_schema, _constraint, _referred_schema: to_schema - ), - ) + + def _referred_schema(_table: sa.Table, _to_schema, _constraint, referred_schema: str | None): + # to_metadata(): None means "unchanged", BLANK_SCHEMA actually clears the schema. + # referred_schema is already validate_schema_tag()-clean (checked below), no + # re-validation needed here. + target = _effective_schema(engine, resolved, referred_schema, db_schema) + return target if target is not None else sa.BLANK_SCHEMA + + return { + id(table): table.to_metadata( + metadata, + # SQLAlchemy's own stub omits None from schema's declared type, + # despite accepting and correctly handling it at runtime + schema=_effective_schema(engine, resolved, validate_schema_tag(table), db_schema), # ty: ignore[invalid-argument-type] + referred_schema_fn=_referred_schema, + ) + for table in Base.metadata.tables.values() + } def _normalized_type(type_: sa.types.TypeEngine[object], dialect: sa.engine.Dialect) -> str: @@ -136,18 +175,75 @@ def _actual_indexes( } +def _expected_index_signature(index: sa.Index, backend: Backend) -> tuple[str, ...]: + """Per-position signature for index: a plain column's name, or the + normalized compiled SQL text of an expression (e.g. ``func.lower(...)``), + matching how the database reflects a functional index back. Expression + normalization is dialect-specific (e.g. Postgres's own catalog inserts + casts as reflection noise), so it's delegated to backend. + """ + signature = [] + for expr in index.expressions: + if isinstance(expr, sa.Column): + signature.append(expr.name) + elif isinstance(expr, str): + signature.append(backend.normalize_index_expression(expr)) + else: + compiled = str(expr.compile(compile_kwargs={"literal_binds": True})) + signature.append(backend.normalize_index_expression(compiled)) + return tuple(signature) + + +def _actual_index_signature(actual_index: ReflectedIndex, backend: Backend) -> tuple[str, ...]: + """Per-position signature for a reflected index, matching + :func:`_expected_index_signature`'s shape. + """ + column_names = actual_index.get("column_names") or [] + if "expressions" not in actual_index: + return tuple(name for name in column_names if name is not None) + expressions = iter(actual_index.get("expressions") or []) + return tuple( + name if name is not None else backend.normalize_index_expression(next(expressions)) + for name in column_names + ) + + def reconcile_schema( engine: sa.Engine, *, + resolved: ResolvedDatabase | None = None, db_schema: str | None = None, vocabulary_included: bool = False, ) -> SchemaReconciliationReport: - """Compare ORM metadata against the live database schema. Reports missing columns, indexes, FKs, and cluster state.""" + """Compare ORM metadata against the live database schema. + + Parameters + ---------- + engine : sqlalchemy.Engine + Engine to inspect. Its dialect selects the backend used for + cluster-state checks. + resolved : ResolvedDatabase, optional + When given, qualifies each table to its own schema tag + (schema_name/vocab_schema/results_schema) instead of applying + db_schema to every table regardless of tag. + db_schema : str, optional + Blanket schema applied to every table when resolved is not given. + vocabulary_included : bool, optional + Whether vocabulary tables are included in the diff. + + Returns + ------- + SchemaReconciliationReport + Per-table status plus every column, index, FK, and cluster issue + found. + """ excluded_categories: tuple[TableCategory, ...] = ( () if vocabulary_included else (TableCategory.VOCABULARY,) ) _backend = resolve_backend(engine) + _cross_schema_fk_supported = supports_schemas(engine) selected_tables = select_maintenance_tables(exclude_categories=excluded_categories) + schema_qualified_tables = _schema_qualified_tables(engine, resolved, db_schema) inspector = sa.inspect(engine) all_issues: list[ReconciliationIssue] = [] table_results: list[TableReconciliationResult] = [] @@ -155,8 +251,43 @@ def reconcile_schema( with engine.connect() as connection: for maintenance_table in selected_tables: table_issues: list[ReconciliationIssue] = [] - exists = inspector.has_table(maintenance_table.table_name, schema=db_schema) + table_schema_tag = validate_schema_tag(maintenance_table.table) + if table_schema_tag is None: + raise TypeError(f"{maintenance_table.table_name}: table has no schema tag.") + table_schema = _effective_schema(engine, resolved, table_schema_tag, db_schema) + exists = inspector.has_table(maintenance_table.table_name, schema=table_schema) if not exists: + relocated_to = find_table_in_other_schemas( + engine, maintenance_table.table_name, physical_schema=table_schema + ) + if relocated_to: + detail = ( + f"Table is absent from schema {table_schema!r} but found in " + f"{', '.join(sorted(relocated_to))!r}." + ) + table_issues.append( + ReconciliationIssue( + table_name=maintenance_table.table_name, + category=maintenance_table.category, + component="table", + object_name=maintenance_table.table_name, + status=Status.RELOCATED, + expected=table_schema, + actual=", ".join(sorted(relocated_to)), + detail=detail, + ) + ) + table_results.append( + TableReconciliationResult( + table_name=maintenance_table.table_name, + category=maintenance_table.category, + status=Status.RELOCATED, + issue_count=1, + detail=detail, + ) + ) + all_issues.extend(table_issues) + continue table_issues.append( ReconciliationIssue( table_name=maintenance_table.table_name, @@ -181,14 +312,14 @@ def reconcile_schema( all_issues.extend(table_issues) continue - expected_table = _schema_table(maintenance_table.table, db_schema) + expected_table = schema_qualified_tables[id(maintenance_table.table)] expected_columns = {column.name: column for column in expected_table.columns} actual_columns = { str(column["name"]): column - for column in inspector.get_columns(maintenance_table.table_name, schema=db_schema) + for column in inspector.get_columns(maintenance_table.table_name, schema=table_schema) } actual_pk_names = tuple( - inspector.get_pk_constraint(maintenance_table.table_name, schema=db_schema).get("constrained_columns") or [] + inspector.get_pk_constraint(maintenance_table.table_name, schema=table_schema).get("constrained_columns") or [] ) expected_pk_names = tuple(column.name for column in expected_table.primary_key.columns) @@ -272,10 +403,21 @@ def reconcile_schema( ) expected_fks = _expected_foreign_keys(expected_table) - actual_fks = _actual_foreign_keys(inspector, maintenance_table.table_name, db_schema) + actual_fks = _actual_foreign_keys(inspector, maintenance_table.table_name, table_schema) + # Uses the unqualified table, since two differently-tagged tables can collapse to + # the same schema (e.g. all-None on SQLite) and hide a genuine cross-tag FK. + raw_expected_fks = _expected_foreign_keys(maintenance_table.table) for signature, constraint in expected_fks.items(): if signature not in actual_fks: + raw_constraint = raw_expected_fks.get(signature) + if ( + not _cross_schema_fk_supported + and raw_constraint is not None + and validate_schema_tag(raw_constraint.referred_table) != table_schema_tag + ): + # SQLite can never create an inline FK crossing a schema boundary. + continue constrained_columns, referred_table, referred_columns = signature table_issues.append( ReconciliationIssue( @@ -307,7 +449,7 @@ def reconcile_schema( ) expected_idxs = _expected_indexes(expected_table) - actual_idxs = _actual_indexes(inspector, maintenance_table.table_name, db_schema) + actual_idxs = _actual_indexes(inspector, maintenance_table.table_name, table_schema) actual_index_list = list(actual_idxs.values()) renamed_actual_names: set[str] = set() @@ -350,9 +492,9 @@ def reconcile_schema( continue actual_index = actual_idxs[index_name] - expected_columns_for_index = tuple(column.name for column in index.columns) - actual_columns_for_index = tuple(c for c in (actual_index.get("column_names") or []) if c is not None) - if expected_columns_for_index != actual_columns_for_index: + expected_signature = _expected_index_signature(index, _backend) + actual_signature = _actual_index_signature(actual_index, _backend) + if expected_signature != actual_signature: table_issues.append( ReconciliationIssue( table_name=maintenance_table.table_name, @@ -360,8 +502,8 @@ def reconcile_schema( component="index", object_name=index_name, status=Status.MISMATCH, - expected=", ".join(expected_columns_for_index), - actual=", ".join(actual_columns_for_index) if actual_columns_for_index else None, + expected=", ".join(expected_signature), + actual=", ".join(actual_signature) if actual_signature else None, detail="Index columns differ from ORM metadata.", ) ) @@ -384,7 +526,7 @@ def reconcile_schema( actual_cluster = _backend.get_clustered_index_name( connection, maintenance_table.table_name, - db_schema, + schema_tag=table_schema_tag, ) if expected_cluster != actual_cluster: # May be a rename, not drift, so treat like a renamed index. diff --git a/omop_alchemy/maintenance/cli_schema_summary.py b/omop_alchemy/maintenance/cli_schema_summary.py index 14e7f45..89fe89c 100644 --- a/omop_alchemy/maintenance/cli_schema_summary.py +++ b/omop_alchemy/maintenance/cli_schema_summary.py @@ -6,7 +6,8 @@ import sqlalchemy as sa -from .tables import TableCategory, qualified_table_name, select_omop_tables +from oa_configurator import qualified, physical_schema_of +from .tables import TableCategory, select_omop_tables @dataclass(frozen=True) @@ -15,6 +16,7 @@ class TableSummaryResult: table_name: str category: TableCategory + schema_tag: str model_name: str primary_key_columns: tuple[str, ...] exists: bool @@ -24,7 +26,6 @@ class TableSummaryResult: def collect_data_summary( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, existing_only: bool = True, ) -> list[TableSummaryResult]: @@ -35,7 +36,7 @@ def collect_data_summary( results: list[TableSummaryResult] = [] with engine.connect() as connection: for table in tables: - exists = inspector.has_table(table.table_name, schema=db_schema) + exists = inspector.has_table(table.table_name, schema=physical_schema_of(engine, schema_tag=table.schema_tag)) if not exists and existing_only: continue @@ -44,7 +45,7 @@ def collect_data_summary( row_count = int( connection.execute( sa.text( - f"SELECT COUNT(*) FROM {qualified_table_name(table.table_name, db_schema)}" + f"SELECT COUNT(*) FROM {qualified(connection, table.table_name, physical_schema=physical_schema_of(connection, schema_tag=table.schema_tag))}" ) ).scalar_one() ) @@ -53,6 +54,7 @@ def collect_data_summary( TableSummaryResult( table_name=table.table_name, category=table.category, + schema_tag=table.schema_tag, model_name=table.model_name, primary_key_columns=table.primary_key_names, exists=exists, diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index 83f2473..1fef114 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -2,20 +2,34 @@ from __future__ import annotations +from collections.abc import Iterable +from contextlib import ExitStack from dataclasses import dataclass import sqlalchemy as sa -from ._cli_utils import Status, dry_label, dry_status, ensure_schema, reject_reserved_schema +from oa_configurator import ( + ResolvedCDMDatabase, + Role, + ensure_schema, + guard_schema_provenance_for, + physical_schema_of, + validate_schema_tag, +) +from orm_loader.helpers import Base +from ._cli_utils import Status, dry_label, dry_status from .tables import ( MaintenanceTable, TableCategory, - collect_maintenance_tables, missing_maintenance_tables, - schema_adjusted_metadata, ) +def _distinct_schema_tags(tables: Iterable[sa.Table]) -> set[str]: + """Every distinct schema tag among tables. Skips untagged tables.""" + return {tag for table in tables if (tag := validate_schema_tag(table)) is not None} + + @dataclass(frozen=True) class TableCreationResult: """Outcome of attempting to create one missing ORM-managed table from SQLAlchemy metadata.""" @@ -42,14 +56,11 @@ def _table_dependencies(table: MaintenanceTable) -> tuple[str, ...]: def collect_missing_tables( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = True, ) -> list[MaintenanceTable]: - """Return ORM-managed tables that are absent from the target database.""" - inspector = sa.inspect(engine) + """Return ORM-managed tables that are absent from the target database, each checked against its own role's schema.""" return missing_maintenance_tables( - inspector, - db_schema=db_schema, + engine, vocabulary_included=vocabulary_included, ) @@ -57,21 +68,47 @@ def collect_missing_tables( def create_missing_tables( engine: sa.Engine, *, - db_schema: str | None = None, + vocab_engine: sa.Engine | None = None, vocabulary_included: bool = True, dry_run: bool = False, + resolved: ResolvedCDMDatabase | None = None, ) -> list[TableCreationResult]: - """Create any ORM-managed tables missing from the target database. Skips tables with unresolved FK dependencies.""" - reject_reserved_schema(db_schema) + """Create any ORM-managed tables missing from the target database. Skips tables with unresolved FK dependencies. + + Parameters + ---------- + vocab_engine : sqlalchemy.Engine, optional + Engine for vocab-role tables, when ``vocab_connection`` names a + physically different server than ``engine``. Defaults to ``engine`` + (the common, same-connection case). + resolved : ResolvedCDMDatabase, optional + Enables the schema-provenance guard around each ``create_all()`` + call. Omitted by direct test/programmatic callers that hand in a + bare engine with no resolved config behind it, in which case the + guard no-ops. A role whose connection is test_only=true also + no-ops, at the guard's own discretion. + """ + vocab_engine = vocab_engine if vocab_engine is not None else engine if not dry_run: - ensure_schema(engine, db_schema) + ensure_schema(engine, physical_schema_of(engine, schema_tag=Role.PRIMARY)) + # create_all() would fail for a non-existing schema on a fresh database. + if resolved is not None: + # Ensure schemas in split-engined deployments + for schema_tag in _distinct_schema_tags(Base.metadata.tables.values()): + # Primary schema is already ensure above + if schema_tag == Role.PRIMARY.value: + continue + target_engine = vocab_engine if schema_tag == Role.VOCAB.value else engine + ensure_schema(target_engine, physical_schema_of(target_engine, schema_tag=schema_tag)) inspector = sa.inspect(engine) missing_tables = collect_missing_tables( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) - existing_table_names = set(inspector.get_table_names(schema=db_schema)) + # Checking only primary schema would hide existing tables elsewhere, wrongly blocking dependents. + existing_table_names: set[str] = set() + for schema_tag in _distinct_schema_tags(Base.metadata.tables.values()): + existing_table_names |= set(inspector.get_table_names(schema=physical_schema_of(engine, schema_tag=schema_tag))) missing_table_names = {table.table_name for table in missing_tables} blocked_dependencies: dict[str, tuple[str, ...]] = {} @@ -92,36 +129,61 @@ def create_missing_tables( ] results: list[TableCreationResult] = [] - with engine.begin() as connection: - if creatable_tables and not dry_run: - metadata, adjusted_tables = schema_adjusted_metadata( - collect_maintenance_tables(), - db_schema=db_schema, - ) - metadata.create_all( - bind=connection, - tables=[adjusted_tables[table.table_name] for table in creatable_tables], - checkfirst=True, - ) - - for maintenance_table in missing_tables: - blocked = blocked_dependencies.get(maintenance_table.table_name) - results.append( - TableCreationResult( - table_name=maintenance_table.table_name, - category=maintenance_table.category, - model_name=maintenance_table.model_name, - status=( - Status.BLOCKED - if blocked is not None - else dry_status(dry_run, applied=Status.CREATED) - ), - detail=( - "table blocked by unresolved dependencies: " + ", ".join(blocked) - if blocked is not None - else dry_label(dry_run, "table would be created from ORM metadata", "table created from ORM metadata") - ), + if creatable_tables and not dry_run: + all_tables = [table.table for table in creatable_tables] + vocab_tables = [table for table in all_tables if table.schema == Role.VOCAB.value] + other_tables = [table for table in all_tables if table.schema != Role.VOCAB.value] + if vocab_engine is engine: + # One call: create_all's dependency sort and FK-deferral must see every table together. + with engine.begin() as connection, ExitStack() as guards: + for schema_tag in _distinct_schema_tags(all_tables): + tables_for_tag = [table for table in all_tables if table.schema == schema_tag] + guards.enter_context( + guard_schema_provenance_for(connection, resolved, schema_tag=schema_tag, tables=tables_for_tag) + ) + Base.metadata.create_all( + bind=connection, tables=all_tables, checkfirst=True ) + else: + # Split physical connections: a cross-boundary FK can't be created here at all; + # that failure surfaces from create_all itself rather than being masked. + if other_tables: + with engine.begin() as connection, ExitStack() as guards: + for schema_tag in _distinct_schema_tags(other_tables): + tables_for_tag = [table for table in other_tables if table.schema == schema_tag] + guards.enter_context( + guard_schema_provenance_for(connection, resolved, schema_tag=schema_tag, tables=tables_for_tag) + ) + Base.metadata.create_all( + bind=connection, tables=other_tables, checkfirst=True + ) + if vocab_tables: + with ( + vocab_engine.begin() as vocab_connection, + guard_schema_provenance_for(vocab_connection, resolved, schema_tag=Role.VOCAB, tables=vocab_tables), + ): + Base.metadata.create_all( + bind=vocab_connection, tables=vocab_tables, checkfirst=True + ) + + for maintenance_table in missing_tables: + blocked = blocked_dependencies.get(maintenance_table.table_name) + results.append( + TableCreationResult( + table_name=maintenance_table.table_name, + category=maintenance_table.category, + model_name=maintenance_table.model_name, + status=( + Status.BLOCKED + if blocked is not None + else dry_status(dry_run, applied=Status.CREATED) + ), + detail=( + "table blocked by unresolved dependencies: " + ", ".join(blocked) + if blocked is not None + else dry_label(dry_run, "table would be created from ORM metadata", "table created from ORM metadata") + ), ) + ) return results diff --git a/omop_alchemy/maintenance/cli_tables.py b/omop_alchemy/maintenance/cli_tables.py index 7c7fa5e..e24678a 100644 --- a/omop_alchemy/maintenance/cli_tables.py +++ b/omop_alchemy/maintenance/cli_tables.py @@ -7,11 +7,19 @@ import sqlalchemy as sa import typer +from oa_configurator import ( + SCHEMA_TRANSLATE_MAP_KEY, + ResolvedCDMDatabase, + Role, + autocommit_connection, + guard_schema_provenance_for, + physical_schema_of, + qualified, +) from ..backends import resolve_backend, require_backend_support, backend_support_note -from ._cli_utils import Status, dry_label, dry_status, omop_command, reject_reserved_schema, resolve_selection +from ._cli_utils import Status, dry_label, dry_status, omop_command, resolve_selection from .tables import ( TableCategory, - qualified_table_name, resolve_maintenance_tables, select_omop_tables, ) @@ -39,6 +47,7 @@ class AnalyzeTableResult: table_name: str category: TableCategory + schema_tag: str operation: str status: Status detail: str @@ -47,7 +56,6 @@ class AnalyzeTableResult: def analyze_tables( engine: sa.Engine, *, - db_schema: str | None = None, scope: TableCategory | None = None, table_names: tuple[str, ...] | None = None, vacuum: bool = False, @@ -57,7 +65,6 @@ def analyze_tables( Runs on every ORM-managed table if both scope and table_names are omitted. """ - reject_reserved_schema(db_schema) if scope is not None and table_names is not None: raise RuntimeError("Use either `scope` or `table_names`, not both.") @@ -67,19 +74,17 @@ def analyze_tables( operation = "VACUUM ANALYZE" if vacuum else "ANALYZE" results: list[AnalyzeTableResult] = [] - connection_factory = ( - engine.connect().execution_options(isolation_level="AUTOCOMMIT") - if vacuum - else engine.connect() - ) + connection_factory = autocommit_connection(engine) if vacuum else engine.connect() with connection_factory as connection: for maintenance_table in selected_tables: - if not inspector.has_table(maintenance_table.table_name, schema=db_schema): + table_schema = physical_schema_of(engine, schema_tag=maintenance_table.schema_tag) + if not inspector.has_table(maintenance_table.table_name, schema=table_schema): results.append( AnalyzeTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, + schema_tag=maintenance_table.schema_tag, operation=operation, status=Status.SKIPPED, detail="table not present in target database", @@ -88,12 +93,15 @@ def analyze_tables( continue if not dry_run: - backend.analyze_table(connection, maintenance_table.table_name, db_schema, vacuum=vacuum) + backend.analyze_table( + connection, maintenance_table.table_name, vacuum=vacuum, schema_tag=maintenance_table.schema_tag + ) results.append( AnalyzeTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, + schema_tag=maintenance_table.schema_tag, operation=operation, status=dry_status(dry_run), detail=dry_label(dry_run, f"{operation.lower()} would run", f"{operation.lower()} completed"), @@ -113,29 +121,51 @@ class TruncateTableResult: table_name: str category: TableCategory + schema_tag: str row_count: int | None status: Status detail: str +def _known_schema_tags(engine: sa.Engine) -> set[str]: + """Schema tags engine's own schema_translate_map actually routes, plus primary. + + Deliberately not oa_configurator.registered_schema_tags(): that's every tag + any package anywhere has registered, including ones for a wholly separate + database (e.g. omop_emb's "registry") that this engine has no entry for and + can't meaningfully resolve -- scanning those would check a schema that not + only doesn't exist but was never this engine's concern to begin with. + """ + stm = engine.get_execution_options().get(SCHEMA_TRANSLATE_MAP_KEY) or {} + return set(stm) | {Role.PRIMARY.value} + + def _blocking_foreign_key_references( + engine: sa.Engine, inspector: sa.Inspector, *, - db_schema: str | None, selected_table_names: set[str], ) -> dict[str, set[str]]: - """Return tables outside the selection that FK-reference at least one selected table, preventing truncation.""" - blockers: dict[str, set[str]] = {} + """Return tables outside the selection that FK-reference at least one selected table, preventing truncation. - for table_name in inspector.get_table_names(schema=db_schema): - if table_name in selected_table_names: - continue + A blocking table can live under any schema tag engine itself routes (a + vocab table can FK-reference a clinical table's PK, or vice versa; + likewise an extension table), so every one of those is scanned, not + just the primary one. + """ + blockers: dict[str, set[str]] = {} - for foreign_key in inspector.get_foreign_keys(table_name, schema=db_schema): - referred_table = foreign_key.get("referred_table") - if referred_table not in selected_table_names: + for schema_tag in _known_schema_tags(engine): + tag_schema = physical_schema_of(engine, schema_tag=schema_tag) + for table_name in inspector.get_table_names(schema=tag_schema): + if table_name in selected_table_names: continue - blockers.setdefault(str(referred_table), set()).add(table_name) + + for foreign_key in inspector.get_foreign_keys(table_name, schema=tag_schema): + referred_table = foreign_key.get("referred_table") + if referred_table not in selected_table_names: + continue + blockers.setdefault(str(referred_table), set()).add(table_name) return blockers @@ -160,15 +190,14 @@ def _format_blocking_reference_error(blockers: dict[str, set[str]]) -> str: def truncate_tables( engine: sa.Engine, *, - db_schema: str | None = None, scope: TableCategory | None = None, table_names: tuple[str, ...] | None = None, restart_identities: bool = False, cascade: bool = False, dry_run: bool = False, + resolved: ResolvedCDMDatabase | None = None, ) -> list[TruncateTableResult]: """Truncate selected ORM-managed tables. Raises if non-selected tables hold blocking FK references.""" - reject_reserved_schema(db_schema) if scope is not None and table_names is not None: raise RuntimeError("Use either `scope` or `table_names`, not both.") if scope is None and table_names is None: @@ -177,17 +206,22 @@ def truncate_tables( backend = resolve_backend(engine) require_backend_support(backend, "truncate_table_batch", "Table truncation") selected_tables = resolve_maintenance_tables(scope=scope, table_names=table_names) + tables_by_name = {table.table_name: table for table in selected_tables} inspector = sa.inspect(engine) results: list[TruncateTableResult] = [] existing_tables: list[str] = [] + existing_table_names_by_schema_tag: dict[str, list[str]] = {} with engine.begin() as connection: for maintenance_table in selected_tables: - if not inspector.has_table(maintenance_table.table_name, schema=db_schema): + if not inspector.has_table( + maintenance_table.table_name, schema=physical_schema_of(engine, schema_tag=maintenance_table.schema_tag) + ): results.append( TruncateTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, + schema_tag=maintenance_table.schema_tag, row_count=None, status=Status.SKIPPED, detail="table not present in target database", @@ -197,14 +231,18 @@ def truncate_tables( row_count = int( connection.exec_driver_sql( - f"SELECT COUNT(*) FROM {qualified_table_name(maintenance_table.table_name, db_schema)}" + f"SELECT COUNT(*) FROM {qualified(connection, maintenance_table.table_name, physical_schema=physical_schema_of(connection, schema_tag=maintenance_table.schema_tag))}" ).scalar_one() ) existing_tables.append(maintenance_table.table_name) + existing_table_names_by_schema_tag.setdefault(maintenance_table.schema_tag, []).append( + maintenance_table.table_name + ) results.append( TruncateTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, + schema_tag=maintenance_table.schema_tag, row_count=row_count, status=dry_status(dry_run), detail=dry_label(dry_run, "table would be truncated", "table truncated"), @@ -213,21 +251,30 @@ def truncate_tables( if existing_tables and not dry_run and not cascade: blockers = _blocking_foreign_key_references( + engine, inspector, - db_schema=db_schema, selected_table_names=set(existing_tables), ) if blockers: raise RuntimeError(_format_blocking_reference_error(blockers)) if existing_tables and not dry_run: - backend.truncate_table_batch( - connection, - existing_tables, - db_schema, - restart_identities=restart_identities, - cascade=cascade, - ) + # One TRUNCATE batch per schema_tag, since truncate_table_batch qualifies its whole list with a single tag. + for schema_tag, table_names_for_tag in existing_table_names_by_schema_tag.items(): + # A same-named table could already exist under a drifted schema, so truncate could hit unrelated data. + with guard_schema_provenance_for( + connection, + resolved, + schema_tag=schema_tag, + tables=[tables_by_name[name].table for name in table_names_for_tag], + ): + backend.truncate_table_batch( + connection, + table_names_for_tag, + restart_identities=restart_identities, + cascade=cascade, + schema_tag=schema_tag, + ) return results @@ -242,6 +289,7 @@ class SequenceTarget: table_name: str category: TableCategory + schema_tag: str pk_column_name: str @@ -251,6 +299,7 @@ class SequenceResetResult: table_name: str category: TableCategory + schema_tag: str pk_column_name: str sequence_name: str | None next_value: int | None @@ -275,6 +324,7 @@ def collect_sequence_targets( SequenceTarget( table_name=table.table_name, category=table.category, + schema_tag=table.schema_tag, pk_column_name=pk_column_name, ) ) @@ -284,12 +334,10 @@ def collect_sequence_targets( def reset_model_sequences( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, dry_run: bool = False, ) -> list[SequenceResetResult]: """Reset each owned sequence to MAX(pk_column) + 1 to prevent insert conflicts after bulk loads.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "find_sequence_name", "Sequence reset") inspector = sa.inspect(engine) @@ -298,11 +346,11 @@ def reset_model_sequences( with engine.begin() as connection: for target in targets: - if not inspector.has_table(target.table_name, schema=db_schema): + if not inspector.has_table(target.table_name, schema=physical_schema_of(engine, schema_tag=target.schema_tag)): continue sequence_name = backend.find_sequence_name( - connection, target.table_name, target.pk_column_name, db_schema + connection, target.table_name, target.pk_column_name, schema_tag=target.schema_tag ) if sequence_name is None: @@ -310,6 +358,7 @@ def reset_model_sequences( SequenceResetResult( table_name=target.table_name, category=target.category, + schema_tag=target.schema_tag, pk_column_name=target.pk_column_name, sequence_name=None, next_value=None, @@ -319,7 +368,9 @@ def reset_model_sequences( ) continue - fully_qualified = qualified_table_name(target.table_name, db_schema) + fully_qualified = qualified( + connection, target.table_name, physical_schema=physical_schema_of(connection, schema_tag=target.schema_tag) + ) current_max = connection.execute( sa.text( f"SELECT COALESCE(MAX({target.pk_column_name}), 0) " @@ -335,6 +386,7 @@ def reset_model_sequences( SequenceResetResult( table_name=target.table_name, category=target.category, + schema_tag=target.schema_tag, pk_column_name=target.pk_column_name, sequence_name=sequence_name, next_value=next_value, @@ -380,7 +432,6 @@ def analyze_tables_command( with console.status("Refreshing planner statistics for selected tables..."): results = analyze_tables( engine, - db_schema=conn.db_schema, scope=resolved_scope, table_names=resolved_tables, vacuum=vacuum, @@ -410,7 +461,6 @@ def reset_sequences_command( with console.status("Resetting PostgreSQL sequences..."): results = reset_model_sequences( engine, - db_schema=conn.db_schema, vocabulary_included=vocabulary_included, dry_run=dry_run, ) @@ -469,12 +519,12 @@ def truncate_tables_command( with console.status("Truncating selected tables..."): results = truncate_tables( engine, - db_schema=conn.db_schema, scope=resolved_scope, table_names=resolved_tables, restart_identities=restart_identities, cascade=cascade, dry_run=dry_run, + resolved=conn.resolved, ) console.print(render_truncate_results(results)) console.print( diff --git a/omop_alchemy/maintenance/cli_vocab.py b/omop_alchemy/maintenance/cli_vocab.py index d336c10..a335752 100644 --- a/omop_alchemy/maintenance/cli_vocab.py +++ b/omop_alchemy/maintenance/cli_vocab.py @@ -4,17 +4,18 @@ import time from collections.abc import Callable +from contextlib import ExitStack from dataclasses import dataclass from pathlib import Path from typing import Literal, TypeAlias, cast import sqlalchemy as sa import sqlalchemy.orm as so -import sqlalchemy.event as sae from sqlalchemy.exc import OperationalError import typer -from sqlalchemy.pool import NullPool -from orm_loader.backends import resolve_backend +from oa_configurator import ResolvedCDMDatabase, ensure_schema, guard_schema_provenance_for +from orm_loader.backends import STAGING_SCHEMA, resolve_backend +from orm_loader.helpers import Base from orm_loader.tables.typing import CSVTableProtocol from rich.progress import ( BarColumn, @@ -25,7 +26,6 @@ TimeElapsedColumn, ) -from ..backends.resolve import SupportedDialect from omop_alchemy.cdm.model.vocabulary import ( Concept, Concept_Ancestor, @@ -39,17 +39,12 @@ Vocabulary, ) -from ._cli_utils import ( - ReservedSchema, - Status, - ensure_schema, - omop_command, - reject_reserved_schema, -) +from ..backends import backend_supports, resolve_backend as resolve_omop_backend +from ._cli_utils import Status, omop_command from .cli_foreign_keys import manage_foreign_key_triggers from .cli_indexes import manage_indexes from .cli_tables import reset_model_sequences -from .tables import TableCategory, schema_adjusted_metadata, select_maintenance_tables +from .tables import TableCategory, select_maintenance_tables from .ui import ( console, render_error, @@ -247,6 +242,7 @@ def _create_missing_vocabulary_tables( connection: sa.Connection, *, db_schema: str | None, + resolved: ResolvedCDMDatabase | None = None, ) -> int: """Create any vocabulary-category ORM tables that are absent from the target database. Returns the count created.""" vocab_tables = select_maintenance_tables( @@ -261,21 +257,29 @@ def _create_missing_vocabulary_tables( if not missing_tables: return 0 - metadata, adjusted_tables = schema_adjusted_metadata( - vocab_tables, - db_schema=db_schema, - ) - metadata.create_all( - bind=connection, - tables=[adjusted_tables[table.table_name] for table in missing_tables], - checkfirst=True, - ) + tables_by_schema_tag: dict[str, list[sa.Table]] = {} + for table in missing_tables: + tables_by_schema_tag.setdefault(table.schema_tag, []).append(table.table) + + with ExitStack() as guard_stack: + # One provenance guard per schema_tag (count only known at runtime); ExitStack defers every write until the block below succeeds. + for schema_tag, tables in tables_by_schema_tag.items(): + guard_stack.enter_context( + guard_schema_provenance_for(connection, resolved, schema_tag=schema_tag, tables=tables) + ) + Base.metadata.create_all( + bind=connection, + tables=[table.table for table in missing_tables], + checkfirst=True, + ) return len(missing_tables) def load_vocab_source( engine: sa.Engine, *, + vocab_engine: sa.Engine | None = None, + vocab_schema: str | None = None, source_path: str | Path, tables: list[str] | None = None, db_schema: str | None = None, @@ -286,6 +290,7 @@ def load_vocab_source( bulk_mode: bool = True, merge_batch_size: int | None = None, progress_callback: VocabularyLoadProgressCallback | None = None, + resolved: ResolvedCDMDatabase | None = None, ) -> VocabularyLoadReport: """ Load Athena vocabulary CSVs from source_path into the target database. @@ -299,7 +304,25 @@ def load_vocab_source( With bulk_mode (default on PostgreSQL), secondary indexes and FK triggers are toggled globally around the load for speed. Pass --no-bulk-mode when loading a single table to avoid the index drop/rebuild overhead. + + Parameters + ---------- + vocab_engine : sqlalchemy.Engine, optional + Engine to create vocabulary tables on, when ``vocab_connection`` names + a physically different server than ``engine``. Defaults to ``engine``. + CSV loading itself still runs entirely against ``engine``; this only + affects where vocab-tagged tables get created. + vocab_schema : str, optional + Schema vocab-tagged tables live in, for the table-existence check + against ``vocab_engine``. Defaults to ``db_schema``. + resolved : ResolvedCDMDatabase, optional + Enables the schema-provenance guard around any missing vocabulary + table creation. Omitted by direct test/programmatic callers with no + resolved config behind their engine, in which case the guard no-ops. """ + vocab_engine = vocab_engine if vocab_engine is not None else engine + vocab_schema = vocab_schema if vocab_schema is not None else db_schema + resolved_source_path = Path(source_path).expanduser().resolve() if not resolved_source_path.exists() or not resolved_source_path.is_dir(): raise RuntimeError(f"Athena source directory not found: {resolved_source_path}") @@ -333,31 +356,10 @@ def load_vocab_source( + ", ".join(sorted(missing)) ) - reject_reserved_schema(db_schema) - if not dry_run: ensure_schema(engine, db_schema) - ensure_schema(engine, ReservedSchema.STAGING) - - # NullPool: each session/connection is opened fresh and closed immediately after - # use. No stale pooled connections survive between tables, which prevents - # "connection in recovery mode" failures on subsequent tables after a heavy load. - load_engine = sa.create_engine(engine.url, poolclass=NullPool) - if db_schema is not None: - # NullPool discards the DBAPI connection on every commit, so a one-time - # SET search_path on the first checkout doesn't survive into the next - # checkout (e.g. after create_staging_table's commit). Re-apply on every - # new connection via an engine-level connect event so COPY and raw-cursor - # operations always target the right schema. The staging schema no longer - # needs to be on the path because orm-loader now qualifies staging - # table identifiers explicitly. - _quoted_schema = '"' + db_schema.replace('"', '""') + '"' - - @sae.listens_for(load_engine, "connect") - def _set_search_path(dbapi_conn, _record): - cur = dbapi_conn.cursor() - cur.execute(f"SET search_path TO {_quoted_schema}") - cur.close() + ensure_schema(engine, STAGING_SCHEMA) + ensure_schema(vocab_engine, vocab_schema) table_count = sum( 1 @@ -380,7 +382,7 @@ def _set_search_path(dbapi_conn, _record): ) _use_bulk_mode = ( - bulk_mode and not dry_run and engine.dialect.name == SupportedDialect.POSTGRESQL + bulk_mode and not dry_run and backend_supports(resolve_omop_backend(engine), "toggle_fk_triggers") ) if _use_bulk_mode: _emit( @@ -393,7 +395,6 @@ def _set_search_path(dbapi_conn, _record): engine, enable=False, vocabulary_included=True, - db_schema=db_schema, dry_run=False, ) _emit( @@ -406,7 +407,6 @@ def _set_search_path(dbapi_conn, _record): engine, enable=False, vocabulary_included=True, - db_schema=db_schema, dry_run=False, ) index_warnings = tuple( @@ -423,9 +423,9 @@ def _set_search_path(dbapi_conn, _record): ) if not dry_run: - with load_engine.connect() as pre_conn: + with vocab_engine.connect() as pre_conn: created_table_count = _create_missing_vocabulary_tables( - pre_conn, db_schema=db_schema + pre_conn, db_schema=vocab_schema, resolved=resolved ) pre_conn.commit() @@ -481,7 +481,7 @@ def _set_search_path(dbapi_conn, _record): _prev_attempt_was_crash = False for attempt in range(3): try: - with so.Session(load_engine) as session: + with so.Session(engine) as session: if ( _prev_attempt_was_crash and merge_strategy == "insert_if_empty" @@ -509,7 +509,7 @@ def _set_search_path(dbapi_conn, _record): index_strategy="keep" if _use_bulk_mode else "auto", chunksize=chunksize, merge_batch_size=merge_batch_size, - staging_schema=ReservedSchema.STAGING, + staging_schema=STAGING_SCHEMA, ) session.commit() break @@ -559,7 +559,6 @@ def _set_search_path(dbapi_conn, _record): engine, enable=True, vocabulary_included=True, - db_schema=db_schema, dry_run=False, cluster=False, ) @@ -573,7 +572,6 @@ def _set_search_path(dbapi_conn, _record): engine, enable=True, vocabulary_included=True, - db_schema=db_schema, dry_run=False, ) @@ -584,10 +582,9 @@ def _set_search_path(dbapi_conn, _record): table_count=table_count, ) - if not dry_run and engine.dialect.name == SupportedDialect.POSTGRESQL: + if not dry_run and backend_supports(resolve_omop_backend(engine), "find_sequence_name"): sequence_results = reset_model_sequences( engine, - db_schema=db_schema, vocabulary_included=True, dry_run=False, ) @@ -654,7 +651,7 @@ def load_vocab_source_command( staging_chunk_size: int | None = typer.Option( 100_000, help=( - "[Phase 1] Rows per ORM transaction when loading CSV → staging table. " + "Staging load: rows per ORM transaction when loading CSV → staging table. " "Ignored when the PostgreSQL COPY fast-path is active (the default for " "Athena CSVs). Pass 0 to disable chunking entirely." ), @@ -672,7 +669,7 @@ def load_vocab_source_command( merge_batch_size: int | None = typer.Option( None, help=( - "[Phase 2] Rows per transaction when merging staging → target table. " + "Merge: rows per transaction when merging staging → target table. " "Default: None (no pagination — single INSERT per table, fastest for high-RAM systems). " "Set to a positive integer to enable paginated commits for memory-constrained systems; " "note that pagination adds a COUNT query and an index build on the staging table " @@ -693,52 +690,60 @@ def load_vocab_source_command( ) raise typer.Exit(code=1) - with Progress( - SpinnerColumn(), - TextColumn("[bold cyan]{task.description}"), - BarColumn(bar_width=None), - TaskProgressColumn(), - TimeElapsedColumn(), - console=console, - transient=False, - ) as progress: - task_id = progress.add_task( - "Preparing Athena vocabulary load...", total=100.0, completed=0 - ) - completed_tables: list[str] = [] - - def _update_progress(event: VocabularyLoadProgress) -> None: - progress.update( - task_id, completed=event.percent, description=event.description + vocab_engine = conn.resolved.vocab_engine_for(engine) + try: + with Progress( + SpinnerColumn(), + TextColumn("[bold cyan]{task.description}"), + BarColumn(bar_width=None), + TaskProgressColumn(), + TimeElapsedColumn(), + console=console, + transient=False, + ) as progress: + task_id = progress.add_task( + "Preparing Athena vocabulary load...", total=100.0, completed=0 ) - if event.table_done and event.table_name is not None: - completed_tables.append(event.table_name) - row_info = ( - f": [dim]{event.rows_this_table:,} rows[/dim]" - if event.rows_this_table is not None - else "" - ) - progress.console.print( - f"[green]loaded[/green] [bold]{event.table_name}[/bold]{row_info} " - f"({len(completed_tables)}/{event.table_count})" + completed_tables: list[str] = [] + + def _update_progress(event: VocabularyLoadProgress) -> None: + progress.update( + task_id, completed=event.percent, description=event.description ) + if event.table_done and event.table_name is not None: + completed_tables.append(event.table_name) + row_info = ( + f": [dim]{event.rows_this_table:,} rows[/dim]" + if event.rows_this_table is not None + else "" + ) + progress.console.print( + f"[green]loaded[/green] [bold]{event.table_name}[/bold]{row_info} " + f"({len(completed_tables)}/{event.table_count})" + ) - report = load_vocab_source( - engine, - source_path=effective_athena_source, - tables=tables or None, - db_schema=conn.db_schema, - dry_run=dry_run, - merge_strategy=merge_strategy, - quote_mode=quote_mode, - chunksize=None if staging_chunk_size == 0 else staging_chunk_size, - bulk_mode=bulk_mode, - merge_batch_size=merge_batch_size, - progress_callback=_update_progress, - ) - progress.update( - task_id, completed=100.0, description="Athena vocabulary load complete" - ) + report = load_vocab_source( + engine, + vocab_engine=vocab_engine, + vocab_schema=conn.resolved.vocab_schema, + source_path=effective_athena_source, + tables=tables or None, + db_schema=conn.resolved.schema_name, + dry_run=dry_run, + merge_strategy=merge_strategy, + quote_mode=quote_mode, + chunksize=None if staging_chunk_size == 0 else staging_chunk_size, + bulk_mode=bulk_mode, + merge_batch_size=merge_batch_size, + progress_callback=_update_progress, + resolved=conn.resolved, + ) + progress.update( + task_id, completed=100.0, description="Athena vocabulary load complete" + ) + finally: + if vocab_engine is not engine: + vocab_engine.dispose() console.print(render_vocab_load_results(report.results)) console.print(render_vocab_load_summary(report, dry_run=dry_run)) diff --git a/omop_alchemy/maintenance/tables.py b/omop_alchemy/maintenance/tables.py index 04a5cf9..3f815e2 100644 --- a/omop_alchemy/maintenance/tables.py +++ b/omop_alchemy/maintenance/tables.py @@ -5,11 +5,23 @@ from typing import Iterable import sqlalchemy as sa +from oa_configurator import physical_schema_of, validate_schema_tag class TableCategory(StrEnum): """An OMOP CDM table's structural category, carrying its render style. + Represents a logical grouping of tables as defined by the + [OMOP CDM spec](https://ohdsi.github.io/CommonDataModel/). + The grouping is also reflected in the subpackage under + ``omop_alchemy.cdm.model``, where the same logical grouping is used + to organize the ORM classes. + + Notes + ----- + This logical grouping is independent of the physical schema in + which the table's data is stored (its own ``schema_tag``). + Parameters ---------- code : str @@ -45,6 +57,21 @@ class MaintenanceTable: table: sa.Table primary_key_columns: tuple[sa.Column[object], ...] + @property + def schema_tag(self) -> str: + """The schema_translate_map key this table's data physically lives + under, read off its own declared schema tag. + + Independent of TableCategory: category is a logical/folder grouping + (e.g. cohort/cohort_definition are RESULTS-category despite + classifying as "derived" in the CDM sense), schema_tag is where the + table's rows physically live. + """ + tag = validate_schema_tag(self.table) + if tag is None: + raise TypeError(f"{self.table_name}: table has no schema tag.") + return tag + @property def is_vocabulary(self) -> bool: return self.category is TableCategory.VOCABULARY @@ -71,13 +98,6 @@ def has_single_integer_primary_key(self) -> bool: ) -def qualified_table_name(table_name: str, db_schema: str | None) -> str: - if db_schema: - quoted_schema = '"' + db_schema.replace('"', '""') + '"' - return f"{quoted_schema}.{table_name}" - return table_name - - def _mapped_cdm_table_classes() -> Iterable[type]: import omop_alchemy.cdm.model # noqa: F401 from orm_loader.helpers import Base @@ -230,50 +250,47 @@ def select_omop_tables( def existing_maintenance_tables( - inspector: sa.Inspector, + bindable: sa.Engine | sa.Connection, *, - db_schema: str | None, vocabulary_included: bool, require_single_integer_primary_key: bool = False, ) -> list[MaintenanceTable]: + """ORM-managed tables that already exist, each checked against its own schema tag. + + Parameters + ---------- + bindable : sqlalchemy.Engine or sqlalchemy.Connection + Used both to inspect the database and, via its schema_translate_map, + to resolve each table's own schema tag to a physical schema + (``physical_schema_of(bindable, schema_tag=table.schema_tag)``) -- a blanket + schema passed in once would silently misclassify every vocab/results + table checked against a database with a genuine primary/vocab/ + results split. + """ + inspector = sa.inspect(bindable) return [ table for table in select_omop_tables( vocabulary_included=vocabulary_included, require_single_integer_primary_key=require_single_integer_primary_key, ) - if inspector.has_table(table.table_name, schema=db_schema) + if inspector.has_table(table.table_name, schema=physical_schema_of(bindable, schema_tag=table.schema_tag)) ] def missing_maintenance_tables( - inspector: sa.Inspector, + bindable: sa.Engine | sa.Connection, *, - db_schema: str | None, vocabulary_included: bool, ) -> list[MaintenanceTable]: + """ORM-managed tables that are absent, each checked against its own schema tag. + + See :func:`existing_maintenance_tables` for why *bindable* replaces a + single ``db_schema`` string. + """ + inspector = sa.inspect(bindable) return [ table for table in select_omop_tables(vocabulary_included=vocabulary_included) - if not inspector.has_table(table.table_name, schema=db_schema) + if not inspector.has_table(table.table_name, schema=physical_schema_of(bindable, schema_tag=table.schema_tag)) ] - - -def schema_adjusted_metadata( - tables: Iterable[MaintenanceTable], - *, - db_schema: str | None, -) -> tuple[sa.MetaData, dict[str, sa.Table]]: - metadata = sa.MetaData() - adjusted_tables: dict[str, sa.Table] = {} - - for maintenance_table in tables: - adjusted_tables[maintenance_table.table_name] = maintenance_table.table.to_metadata( - metadata, - schema=db_schema, # ty: ignore[invalid-argument-type] - referred_schema_fn=( - lambda _table, to_schema, _constraint, _referred_schema: to_schema - ), - ) - - return metadata, adjusted_tables diff --git a/omop_alchemy/maintenance/ui.py b/omop_alchemy/maintenance/ui.py index ce3364a..e0a2492 100644 --- a/omop_alchemy/maintenance/ui.py +++ b/omop_alchemy/maintenance/ui.py @@ -9,7 +9,7 @@ from rich.table import Table from rich.text import Text -from ..backends.resolve import _DIALECT_TO_BACKEND_MAP, SupportedDialect as _SupportedDialect +from ..backends.resolve import backend_label from .ascii import render_banner from .tables import TableCategory @@ -60,12 +60,6 @@ def _status_style(status: Status) -> str: ) return status.severity.style -def _backend_label(dialect_name: str) -> str: - try: - return _DIALECT_TO_BACKEND_MAP[_SupportedDialect(dialect_name)].name - except (ValueError, KeyError): - return dialect_name - def _bool_label(value: bool) -> Text: return Text("yes" if value else "no", style="green" if value else "dim") @@ -173,7 +167,7 @@ def render_info_database(info: MaintenanceInfo) -> Panel: grid.add_column(style="bold cyan") grid.add_column() grid.add_row("Engine URL", info.engine_url or "-") - grid.add_row("Backend", _backend_label(info.backend) if info.backend else "-") + grid.add_row("Backend", backend_label(info.backend) if info.backend else "-") grid.add_row("Engine created", _bool_label(info.engine_created)) grid.add_row("Connection ready", _bool_label(info.connection_ready)) @@ -215,7 +209,7 @@ def render_backup_result(result: BackupResult) -> Panel: grid.add_column(style="bold cyan") grid.add_column() grid.add_row("Status", _status_text(result.status)) - grid.add_row("Backend", _backend_label(result.backend)) + grid.add_row("Backend", backend_label(result.backend)) grid.add_row("Database", result.database_name) grid.add_row("Schema", result.schema_name or "all schemas") grid.add_row("Format", result.backup_format.value) @@ -249,7 +243,7 @@ def render_restore_result(result: BackupResult) -> Panel: grid.add_column(style="bold cyan") grid.add_column() grid.add_row("Status", _status_text(result.status)) - grid.add_row("Backend", _backend_label(result.backend)) + grid.add_row("Backend", backend_label(result.backend)) grid.add_row("Database", result.database_name) grid.add_row("Schema", result.schema_name or "all schemas") grid.add_row("Format", result.backup_format.value) @@ -348,7 +342,7 @@ def render_reconciliation_summary(report: SchemaReconciliationReport) -> Panel: grid = Table.grid(padding=(0, 2)) grid.add_column(style="bold cyan") grid.add_column() - grid.add_row("Backend", _backend_label(report.backend)) + grid.add_row("Backend", backend_label(report.backend)) grid.add_row("Tables", str(len(report.table_results))) if matched: grid.add_row(Status.MATCHED.value.capitalize(), str(matched)) diff --git a/omop_alchemy/toolkit/analytics/oncology/oncology_drug_exposure.py b/omop_alchemy/toolkit/analytics/oncology/oncology_drug_exposure.py index 8a7af84..4ab81ac 100644 --- a/omop_alchemy/toolkit/analytics/oncology/oncology_drug_exposure.py +++ b/omop_alchemy/toolkit/analytics/oncology/oncology_drug_exposure.py @@ -2,6 +2,8 @@ from sqlalchemy.ext.hybrid import hybrid_property +from oa_configurator import Role + from omop_alchemy.cdm.model.clinical.drug_exposure import Drug_ExposureView from .concept_sets import SACT_DRUGS, resolve_sact_drug_concept_ids @@ -20,6 +22,11 @@ class OncologyDrugExposure(Drug_ExposureView): from one governed ``ConceptGroupSpec``, including its exclusions. """ + __tablename__ = "drug_exposure" + # Must match Drug_Exposure's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} + __mapper_args__ = {"concrete": False} + @hybrid_property def is_sact(self) -> bool: return self.drug_concept_id in resolve_sact_drug_concept_ids( diff --git a/omop_alchemy/toolkit/analytics/oncology/oncology_episodes.py b/omop_alchemy/toolkit/analytics/oncology/oncology_episodes.py index 6bf1fe8..8a25182 100644 --- a/omop_alchemy/toolkit/analytics/oncology/oncology_episodes.py +++ b/omop_alchemy/toolkit/analytics/oncology/oncology_episodes.py @@ -6,6 +6,7 @@ from typing import Self, cast import sqlalchemy.orm as so +from oa_configurator import Role from sqlalchemy.ext.declarative import declared_attr from sqlalchemy.ext.hybrid import hybrid_property from sqlalchemy.orm import object_session @@ -72,6 +73,11 @@ class OncologyEpisode( audit disagreements. """ + __tablename__ = "episode" + # Must match EpisodeView's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} + __mapper_args__ = {"concrete": False} + @declared_attr @classmethod def children(cls) -> so.Mapped[list["OncologyEpisode"]]: diff --git a/omop_alchemy/toolkit/analytics/oncology/oncology_procedure_occurrence.py b/omop_alchemy/toolkit/analytics/oncology/oncology_procedure_occurrence.py index 72b02d8..a0ff288 100644 --- a/omop_alchemy/toolkit/analytics/oncology/oncology_procedure_occurrence.py +++ b/omop_alchemy/toolkit/analytics/oncology/oncology_procedure_occurrence.py @@ -2,6 +2,8 @@ from sqlalchemy.ext.hybrid import hybrid_property +from oa_configurator import Role + from omop_alchemy.cdm.model.clinical.procedure_occurrence import ( Procedure_OccurrenceView, ) @@ -33,6 +35,11 @@ class OncologyProcedure(Procedure_OccurrenceView): determine membership, and reporting "no" would silently misclassify. """ + __tablename__ = "procedure_occurrence" + # Must match Procedure_Occurrence's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} + __mapper_args__ = {"concrete": False} + @hybrid_property def is_radiotherapy(self) -> bool: return self.procedure_concept_id in resolve_rt_procedure_concept_ids( diff --git a/omop_alchemy/toolkit/core/timeline/event_timeline.py b/omop_alchemy/toolkit/core/timeline/event_timeline.py index ca5abce..73e6995 100644 --- a/omop_alchemy/toolkit/core/timeline/event_timeline.py +++ b/omop_alchemy/toolkit/core/timeline/event_timeline.py @@ -5,6 +5,7 @@ Drug_Exposure, Observation, ) +from oa_configurator import Role from sqlalchemy.orm import object_session from sqlalchemy import select from datetime import datetime, time, date @@ -297,6 +298,12 @@ class Condition_Event(ClinicalEvent, Condition_Occurrence): class Measurement_Event(ClinicalEvent, Measurement): + # Measurement's ModifierSourceMixin carries __abstract__ = True; without + # redeclaring these, SQLAlchemy silently builds a second, unlinked Table + # object for a subclass that doesn't otherwise declare its own table. + __tablename__ = "measurement" + __table_args__ = {"schema": Role.PRIMARY.value} + _mapping = EventMapping.from_model( Measurement, value_fields=[ @@ -325,6 +332,12 @@ def event_metadata(self) -> Mapping[str, Any]: class Observation_Event(ClinicalEvent, Observation): + # Observation's ModifierSourceMixin carries __abstract__ = True; without + # redeclaring these, SQLAlchemy silently builds a second, unlinked Table + # object for a subclass that doesn't otherwise declare its own table. + __tablename__ = "observation" + __table_args__ = {"schema": Role.PRIMARY.value} + _mapping = EventMapping.from_model( Observation, value_fields=["value_as_concept_id", "value_as_number", "value_as_string"], diff --git a/omop_alchemy/toolkit/episodes/handling/resolved_event.py b/omop_alchemy/toolkit/episodes/handling/resolved_event.py index 75133eb..69bc75a 100644 --- a/omop_alchemy/toolkit/episodes/handling/resolved_event.py +++ b/omop_alchemy/toolkit/episodes/handling/resolved_event.py @@ -6,6 +6,8 @@ import sqlalchemy.orm as so from sqlalchemy.ext.declarative import declared_attr +from oa_configurator import Role + from omop_alchemy.cdm.base import ModifierFieldConcepts from omop_alchemy.cdm.model.structural.episode_event import Episode_EventView @@ -59,6 +61,11 @@ class declares it, or the target row itself is missing. through ordinary ``episode.episode_events`` traversal. """ + __tablename__ = "episode_event" + # Must match Episode_Event's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} + __mapper_args__ = {"concrete": False} + @classmethod def recognized_field_concept_ids(cls) -> set[int]: return _known_modifier_field_concept_ids() diff --git a/pyproject.toml b/pyproject.toml index 9f247ad..026ca31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,3 +92,5 @@ packages = ["omop_alchemy"] cache-keys = [{ file = "pyproject.toml" }, { git = { commit = true, tags = true } }] [tool.pytest.ini_options] +addopts = "-m 'not db_dialect'" + diff --git a/tests/conftest.py b/tests/conftest.py index 76e8fe6..776f9d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,17 +1,21 @@ import copy +import os from datetime import date from pathlib import Path import pytest import sqlalchemy as sa +import typer.rich_utils as _typer_rich_utils from orm_loader.helpers import bootstrap +from oa_configurator.testing import isolated_test_database, isolated_test_schema +from oa_configurator import ResolvedCDMDatabase, ResolvedConnection, registered_schema_tags import sqlalchemy.orm as so from sqlalchemy.orm import Session, sessionmaker -from typing import Any, Dict, Tuple +from typing import Any, Dict, Iterator, Tuple +from omop_alchemy.config import OmopAlchemyConfig from omop_alchemy.maintenance.cli_vocab import _load_vocab_model_csv -from omop_alchemy.cdm.model.clinical import Condition_Occurrence, Person -from omop_alchemy.cdm.model.derived import Observation_Period +from omop_alchemy.cdm.model.clinical import Condition_Occurrence, Observation_Period, Person from omop_alchemy.cdm.model.structural import Episode, Episode_Event from omop_alchemy.cdm.model.vocabulary import ( Concept, @@ -23,6 +27,63 @@ Vocabulary, ) +# typer forces colorized rich error/output rendering when GITHUB_ACTIONS (or +# FORCE_COLOR / PY_COLORS) is set -- see typer.rich_utils.FORCE_TERMINAL. Under +# GitHub Actions that injects ANSI escapes into CLI output, breaking tests that +# assert on plain-substring message content (e.g. "no such option: --foo"). The +# force feeds every typer rich Console, so clear it here: tests then see the same +# uncolored output everywhere; real users still get colour in a real terminal. +_typer_rich_utils.FORCE_TERMINAL = None + + +def resolved_cdm_database_from_engine( + engine: sa.Engine, + *, + name: str, + schema_name: str | None = None, + vocab_schema: str | None = None, + results_schema: str | None = None, +) -> ResolvedCDMDatabase: + """Build a ResolvedCDMDatabase from an already-live engine's own URL. + + Only for a case with no resolved object to build off at all (e.g. a + bare SQLite engine). When one already exists, prefer + ``dataclasses.replace(existing.resolved, ...)`` instead. + """ + url = engine.url + connection = ResolvedConnection( + name=name, + url=url.render_as_string(hide_password=False), + safe_url=url.render_as_string(hide_password=True), + _engine_url=url, + ) + return ResolvedCDMDatabase( + name=name, + connection=connection, + schema_name=schema_name, + vocab_connection=connection, + vocab_schema=vocab_schema, + results_schema=results_schema, + ) + + +@pytest.fixture +def fresh_engine() -> Iterator[sa.Engine]: + """Fresh, empty, function-scoped SQLite engine. + + SQLite has no schema concept, so every schema tag maps back to None, + matching the flat namespace every caller here has always assumed. + isolated_test_database's own default already folds every registered + tag this way when execution_options omits schema_translate_map. + """ + with isolated_test_database( + OmopAlchemyConfig, + "test_cdm_db_sqlite", + dialect="sqlite", + future=True, + ) as db: + yield db.connection.engine + ATHENA_LOAD_ORDER = [ Domain, @@ -294,71 +355,123 @@ def _seed_basic_clinical_data(session: Session) -> None: @pytest.fixture(scope="session") -def engine(tmp_path_factory: pytest.TempPathFactory): +def engine(tmp_path_factory: pytest.TempPathFactory) -> Iterator[sa.Engine]: """ Session-scoped SQLite engine built from repo fixtures. - The database is created fresh for each test session, so it behaves - the same on the host and inside containers. + Resolves to a real tempfile (not ``:memory:``), matching this fixture's + own need to behave the same on the host and inside containers. """ - db_dir = tmp_path_factory.mktemp("omop-alchemy") - db_path = db_dir / "test.db" - engine = sa.create_engine( - f"sqlite:///{db_path}", + with isolated_test_database( + OmopAlchemyConfig, + "test_cdm_db_sqlite", + dialect="sqlite", future=True, echo=False, poolclass=sa.pool.StaticPool, connect_args={"check_same_thread": False, "timeout": 30}, - ) + # SQLite has no schema concept: isolated_test_database's own default + # folds every registered schema tag back to None, matching the flat + # namespace this fixture has always assumed. + ) as db: + engine = db.connection.engine + bootstrap(engine, create=True) + _load_fixture_vocabulary(engine, tmp_path_factory.mktemp("omop-alchemy-fixtures")) - bootstrap(engine, create=True) - _load_fixture_vocabulary(engine, db_dir) + with so.Session(engine, expire_on_commit=False) as seed_session: + _seed_basic_clinical_data(seed_session) - with so.Session(engine, expire_on_commit=False) as seed_session: - _seed_basic_clinical_data(seed_session) - - try: yield engine - finally: - engine.dispose() -@pytest.fixture(scope="session") -def pg_engine(): - """Session-scoped PostgreSQL engine for integration tests. +@pytest.fixture +def pg_db(request): + """Canonical isolated PostgreSQL test database (Phase 0 of the + schema_translate_map fix). - Resolves via OA_Configurator resource 'test_cdm_db' in ~/.config/omop/config.toml. + Resolves via OA_Configurator resource 'test_cdm_db_pg' in ~/.config/omop/config.toml. Run: omop-config configure omop_alchemy (answer Y when asked to configure test database). + + Everything a test does through ``pg_db.connection``/``pg_db.session`` + happens inside one transaction that's rolled back on exit: nothing + here is ever committed to the shared server, so concurrent test runs + can't collide and no manual cleanup is needed. + + New tests should use this directly rather than ``pg_engine``/``pg_session`` + below, which need a real, genuinely-committing ``Engine`` (for code that + calls ``.connect()``/``.begin()`` on what it's given) and are now thin + shims over this fixture's own connection. """ - from oa_configurator.pytest_plugin import ( - ensure_test_db_exists, - ensure_test_user_exists, - resolve_test_database, - ) + from oa_configurator.testing import isolated_test_database from omop_alchemy.config import OmopAlchemyConfig - url = resolve_test_database(OmopAlchemyConfig, "test_cdm_db") - ensure_test_user_exists(url) - ensure_test_db_exists(url) - engine = sa.create_engine(url, future=True) - try: - yield engine - finally: - engine.dispose() + with isolated_test_database(OmopAlchemyConfig, "test_cdm_db_pg", request=request) as db: + yield db @pytest.fixture -def pg_session(pg_engine): +def pg_engine(pg_db): + """Real, genuinely-committing PostgreSQL engine on the connection's own + default schema (``public``). ``pg_session`` resets it clean before and + after each test; this fixture just hands back the engine to run + genuine engine-building code paths against (``.connect()``/``.begin()``, + which a bare ``Connection`` can't stand in for). + + A thin shim over ``pg_db``'s own ``committing_engine``: every registered + schema tag folds back to the connection's default, matching the + single-schema setup ``pg_session`` provides. """ - Function-scoped PostgreSQL session with a clean schema for each test. + return pg_db.committing_engine.execution_options( + schema_translate_map={tag: None for tag in registered_schema_tags()} + ) + + +_SYSTEM_SCHEMAS = frozenset({"pg_catalog", "information_schema"}) - Drops and recreates the public schema before each test to ensure full isolation. + +def _reset_test_database(engine: sa.Engine) -> None: + """Drop every non-system schema and recreate public. + + Only safe against a database used by one process at a time: this + suite runs sequentially by design (no ``pytest-xdist`` support), so a + single shared schema reset before/after each test is simpler than + per-test isolation and gives the same guarantee here. Fails loudly + rather than racing if that assumption is ever violated (e.g. `-n 2+` + run by mistake). """ - with pg_engine.connect() as conn: - conn.execute(sa.text("DROP SCHEMA public CASCADE")) + worker_count = os.environ.get("PYTEST_XDIST_WORKER_COUNT") + if worker_count is not None and int(worker_count) > 1: + pytest.fail( + "_reset_test_database() cannot run safely under parallel pytest-xdist " + f"workers ({worker_count} active): it drops and recreates every " + "non-system schema in a database shared across the whole test session, " + "so concurrent workers would race each other's resets. This suite is " + "sequential-only; run it without -n." + ) + with engine.connect() as conn: + schema_names = sa.inspect(conn).get_schema_names() + for schema in schema_names: + if schema in _SYSTEM_SCHEMAS or schema.startswith("pg_"): + continue + conn.execute(sa.text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')) + conn.execute(sa.text("DROP SCHEMA IF EXISTS public CASCADE")) conn.execute(sa.text("CREATE SCHEMA public")) conn.commit() + +@pytest.fixture +def pg_session(pg_engine, cleanup_after_test): + """Function-scoped PostgreSQL session with a clean schema for each test. + + Resets every non-system schema (not just public) both before and + after each test, via ``cleanup_after_test``, so a test's own + committed DDL/DML -- in public or a reserved bookkeeping schema like + ``MAINTENANCE_SCHEMA`` -- never depends on some later, unrelated test + to wipe it. + """ + _reset_test_database(pg_engine) + cleanup_after_test(lambda: _reset_test_database(pg_engine)) + bootstrap(pg_engine, create=True) session = so.Session(pg_engine, expire_on_commit=False) @@ -369,6 +482,27 @@ def pg_session(pg_engine): session.close() +@pytest.fixture +def pg_schema_session(pg_db): + """PostgreSQL session bound to a unique committed schema. + + Use this for tests that do not assert the literal ``public`` schema. The + schema is dropped at teardown, so separate test processes cannot reset one + another's objects. + """ + with isolated_test_schema(pg_db.committing_engine, prefix="omop_alchemy") as schema: + engine = pg_db.committing_engine.execution_options( + schema_translate_map={tag: schema for tag in registered_schema_tags()} + ) + bootstrap(engine, create=True) + session = so.Session(engine, expire_on_commit=False) + try: + yield session + finally: + session.rollback() + session.close() + + @pytest.fixture(scope="function") def session(engine) -> Session: # type: ignore """ diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 4465856..8f43451 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,4 +1,3 @@ -import sqlalchemy as sa import pytest from typer.testing import CliRunner @@ -9,18 +8,12 @@ runner = CliRunner() -def _engine(tmp_path): - """Create an isolated SQLite engine for analyze-table tests.""" - return sa.create_engine(f"sqlite:///{tmp_path / 'analyze.db'}", future=True) - - -def test_analyze_tables_runs_on_sqlite(tmp_path): +def test_analyze_tables_runs_on_sqlite(fresh_engine): """Analyze applies successfully on SQLite for selected OMOP tables.""" - engine = _engine(tmp_path) - create_missing_tables(engine, vocabulary_included=True) + create_missing_tables(fresh_engine, vocabulary_included=True) results = analyze_tables( - engine, + fresh_engine, scope=TableCategory.CLINICAL, dry_run=False, ) @@ -31,13 +24,11 @@ def test_analyze_tables_runs_on_sqlite(tmp_path): ) -def test_analyze_tables_rejects_vacuum_on_sqlite(tmp_path): +def test_analyze_tables_rejects_vacuum_on_sqlite(fresh_engine): """VACUUM ANALYZE is rejected on SQLite with a clear runtime error.""" - engine = _engine(tmp_path) - create_missing_tables(engine, vocabulary_included=True) + create_missing_tables(fresh_engine, vocabulary_included=True) with pytest.raises(RuntimeError) as exc_info: - analyze_tables(engine, scope=TableCategory.CLINICAL, vacuum=True) + analyze_tables(fresh_engine, scope=TableCategory.CLINICAL, vacuum=True) assert "not supported by the SQLite backend" in str(exc_info.value) - diff --git a/tests/test_backends_non_default_schema_postgres.py b/tests/test_backends_non_default_schema_postgres.py new file mode 100644 index 0000000..5655303 --- /dev/null +++ b/tests/test_backends_non_default_schema_postgres.py @@ -0,0 +1,135 @@ +"""Non-default-schema Postgres coverage for the backends/ signature refactor (Phase 3.2). + +Every other maintenance-CLI test runs against the default schema, where +``physical_schema_of(conn)`` returning ``None`` and the old ``db_schema=None`` +parameter are indistinguishable. The refactor that dropped explicit +``db_schema`` threading through ``backends/`` (deriving it internally via +``physical_schema_of(conn)`` instead) could pass every existing test while still +being broken for a real non-default schema. This exercises a representative +subset of the refactored surface (FK trigger toggle, index create/drop, +full-text install, sequence reset) against a genuine non-default Postgres +schema, using ``pg_engine`` the same way ``test_db_schema_search_path_on_postgres`` +(``test_load_vocab_postgres.py``) already does. +""" + +from __future__ import annotations + +from typing import Iterator, NamedTuple + +import pytest +import sqlalchemy as sa +from oa_configurator import Role + +from oa_configurator.testing import isolated_test_schema + +from omop_alchemy.maintenance._cli_utils import Status +from omop_alchemy.maintenance.cli_foreign_keys import ( + collect_foreign_key_trigger_status, + manage_foreign_key_triggers, +) +from omop_alchemy.maintenance.cli_fulltext import install_fulltext_columns +from omop_alchemy.maintenance.cli_indexes import manage_indexes +from omop_alchemy.maintenance.cli_schema_tables import create_missing_tables +from omop_alchemy.maintenance.cli_tables import reset_model_sequences + +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + + +class _Scoped(NamedTuple): + engine: sa.Engine + schema: str + + +@pytest.fixture() +def scoped(pg_engine: sa.Engine) -> Iterator[_Scoped]: + """``pg_engine`` scoped to a real, non-default, uniquely-named schema. + + Single-schema deployment: vocab/results fall back to the same schema as + everything else, matching ``ResolvedCDMDatabase``'s own fallback when + ``vocab_schema``/``results_schema`` aren't configured, the exact case + that made the vocab/results-role fix safe for unconfigured deployments. + """ + with isolated_test_schema(pg_engine, prefix="backends_non_default") as schema: + engine = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema, "vocab": schema, "results": schema} + ) + yield _Scoped(engine=engine, schema=schema) + + +def test_fk_trigger_toggle_targets_the_configured_schema(scoped: _Scoped) -> None: + create_missing_tables(scoped.engine, vocabulary_included=True) + + disabled = manage_foreign_key_triggers(scoped.engine, enable=False) + assert disabled + assert all(result.status == Status.APPLIED for result in disabled) + + status_after_disable = { + result.table_name: result + for result in collect_foreign_key_trigger_status(scoped.engine) + } + person_status = status_after_disable["person"] + assert person_status.enabled_trigger_count == 0 + assert person_status.disabled_trigger_count > 0 + + enabled = manage_foreign_key_triggers(scoped.engine, enable=True) + assert all(result.status == Status.APPLIED for result in enabled) + + status_after_enable = { + result.table_name: result + for result in collect_foreign_key_trigger_status(scoped.engine) + } + assert status_after_enable["person"].disabled_trigger_count == 0 + + +def test_index_disable_and_enable_targets_the_configured_schema(scoped: _Scoped) -> None: + create_missing_tables(scoped.engine, vocabulary_included=True) + + disabled = manage_indexes(scoped.engine, enable=False) + assert disabled + assert all(result.status in (Status.APPLIED, Status.SKIPPED) for result in disabled) + + inspector = sa.inspect(scoped.engine) + person_indexes_after_disable = { + idx["name"] for idx in inspector.get_indexes("person", schema=scoped.schema) + } + + enabled = manage_indexes(scoped.engine, enable=True) + assert all(result.status in (Status.APPLIED, Status.SKIPPED) for result in enabled) + + inspector = sa.inspect(scoped.engine) + person_indexes_after_enable = { + idx["name"] for idx in inspector.get_indexes("person", schema=scoped.schema) + } + assert person_indexes_after_enable != person_indexes_after_disable or person_indexes_after_enable + + +def test_fulltext_install_targets_the_configured_schema(scoped: _Scoped) -> None: + create_missing_tables(scoped.engine, vocabulary_included=True) + + results = install_fulltext_columns(scoped.engine) + assert results + assert all(result.status == Status.APPLIED for result in results) + + inspector = sa.inspect(scoped.engine) + for result in results: + columns = { + c["name"] for c in inspector.get_columns(result.table_name, schema=scoped.schema) + } + assert result.vector_column_name in columns + + +def test_sequence_reset_targets_the_configured_schema(scoped: _Scoped) -> None: + create_missing_tables(scoped.engine, vocabulary_included=True) + + results = { + r.table_name: r for r in reset_model_sequences(scoped.engine) + } + person_result = results["person"] + + assert person_result.status == Status.RESET + assert person_result.sequence_name is not None + assert person_result.next_value == 1 + + with scoped.engine.begin() as conn: + next_id = conn.execute(sa.text(f"SELECT nextval('{person_result.sequence_name}')")).scalar_one() + assert next_id == 1 diff --git a/tests/test_concept_groups.py b/tests/test_concept_groups.py index ee55a8d..34c8833 100644 --- a/tests/test_concept_groups.py +++ b/tests/test_concept_groups.py @@ -253,20 +253,16 @@ def test_registered_identity_is_shared_across_engines(session): clear_concept_group_cache() -def test_unregistered_engines_do_not_share(session): +def test_unregistered_engines_do_not_share(session, fresh_engine): """Without an identity, each engine is its own scope. In-memory SQLite lands here deliberately: identical configuration, separate databases, so sharing would serve one database's concept sets for another. """ - other = sa.create_engine("sqlite://") - try: - with so.Session(other) as other_session: - assert concept_group_registry(session) is not concept_group_registry( - other_session - ) - finally: - other.dispose() + with so.Session(fresh_engine) as other_session: + assert concept_group_registry(session) is not concept_group_registry( + other_session + ) def test_connection_bound_sessions_share_their_engine_scope(session): diff --git a/tests/test_config_driver.py b/tests/test_config_driver.py index c402842..fdaefad 100644 --- a/tests/test_config_driver.py +++ b/tests/test_config_driver.py @@ -51,9 +51,7 @@ def test_get_cdm_context_resolves_the_typed_database_field(monkeypatch) -> None: connections={ "cdm": ConnectionConfig(dialect="sqlite", database_name=":memory:") }, - databases={ - "cdm_db": CDMDatabaseConfig(connection="cdm", schema_name="main") - }, + databases={"cdm_db": CDMDatabaseConfig(connection="cdm")}, ) monkeypatch.setattr("omop_alchemy.config.load_stack_config", lambda: stack) @@ -61,7 +59,7 @@ def test_get_cdm_context_resolves_the_typed_database_field(monkeypatch) -> None: assert package_config.cdm_db == "cdm_db" assert isinstance(resolved, ResolvedCDMDatabase) - assert resolved.schema_name == "main" + assert resolved.schema_name is None def test_vocabulary_identity_for_colocated_vocabulary() -> None: diff --git a/tests/test_create_tables.py b/tests/test_create_tables.py index 4dd3484..c213fc0 100644 --- a/tests/test_create_tables.py +++ b/tests/test_create_tables.py @@ -1,29 +1,31 @@ +import dataclasses + import sqlalchemy as sa +import sqlalchemy.orm as so +from oa_configurator import Role, register_reserved_schema_tag +from orm_loader.helpers import Base +from omop_alchemy.maintenance import cli_schema_tables from omop_alchemy.maintenance.cli_schema import collect_missing_tables, create_missing_tables +from omop_alchemy.maintenance.tables import MaintenanceTable, TableCategory - -def _engine(tmp_path): - """Create an isolated SQLite engine for table-creation tests.""" - return sa.create_engine(f"sqlite:///{tmp_path / 'create_tables.db'}", future=True) +from tests.conftest import resolved_cdm_database_from_engine -def test_collect_missing_tables_on_empty_database(tmp_path): +def test_collect_missing_tables_on_empty_database(fresh_engine): """An empty database reports core clinical and vocabulary tables as missing.""" - engine = _engine(tmp_path) - missing = collect_missing_tables(engine) + missing = collect_missing_tables(fresh_engine) table_names = {table.table_name for table in missing} assert "person" in table_names assert "concept" in table_names -def test_create_missing_tables_reports_blocked_tables_when_vocabulary_is_missing(tmp_path): +def test_create_missing_tables_reports_blocked_tables_when_vocabulary_is_missing(fresh_engine): """Non-vocabulary creation reports blocked tables when required vocab tables are excluded.""" - engine = _engine(tmp_path) - results = create_missing_tables(engine, vocabulary_included=False) + results = create_missing_tables(fresh_engine, vocabulary_included=False) - inspector = sa.inspect(engine) + inspector = sa.inspect(fresh_engine) assert results assert not inspector.has_table("concept") result_by_name = { @@ -34,27 +36,65 @@ def test_create_missing_tables_reports_blocked_tables_when_vocabulary_is_missing assert "concept" in result_by_name["person"].detail -def test_create_missing_tables_can_recreate_non_vocabulary_tables_when_dependencies_exist(tmp_path): +def test_create_missing_tables_can_recreate_non_vocabulary_tables_when_dependencies_exist(fresh_engine): """Previously dropped non-vocabulary tables can be recreated when dependencies are present.""" - engine = _engine(tmp_path) - create_missing_tables(engine, vocabulary_included=True) + create_missing_tables(fresh_engine, vocabulary_included=True) - with engine.begin() as connection: + with fresh_engine.begin() as connection: connection.exec_driver_sql("DROP TABLE cdm_source") - results = create_missing_tables(engine, vocabulary_included=False) + results = create_missing_tables(fresh_engine, vocabulary_included=False) - inspector = sa.inspect(engine) + inspector = sa.inspect(fresh_engine) assert any(result.table_name == "cdm_source" and result.status == "created" for result in results) assert inspector.has_table("cdm_source") assert inspector.has_table("concept") -def test_create_missing_tables_can_create_vocabulary(tmp_path): +def test_create_missing_tables_can_create_vocabulary(fresh_engine): """Including vocabulary creates both clinical and vocabulary tables.""" - engine = _engine(tmp_path) - create_missing_tables(engine, vocabulary_included=True) + create_missing_tables(fresh_engine, vocabulary_included=True) - inspector = sa.inspect(engine) + inspector = sa.inspect(fresh_engine) assert inspector.has_table("person") assert inspector.has_table("concept") + + +def test_create_missing_tables_creates_table_under_a_non_role_schema_tag(fresh_engine, monkeypatch): + """A table tagged with a registered schema tag outside Role is created, + not silently skipped by a Role-only enumeration. + + Uses a fake MaintenanceTable: tagging a real CDM table this way needs + extension-table support, not yet built on this branch. + """ + register_reserved_schema_tag("synthetic_tag", owner="test") + engine = fresh_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: None, "vocab": None, "results": None, "synthetic_tag": None} + ) + resolved = resolved_cdm_database_from_engine(engine, name="synthetic_tag_test") + test_connection = dataclasses.replace(resolved.connection, test_only=True) + resolved = dataclasses.replace(resolved, connection=test_connection, vocab_connection=test_connection) + + class _SyntheticTagTable(Base): + __tablename__ = "synthetic_tag_table" + __table_args__ = {"schema": "synthetic_tag"} + id: so.Mapped[int] = so.mapped_column(primary_key=True) + + fake_table = MaintenanceTable( + table_name="synthetic_tag_table", + model_name="_SyntheticTagTable", + model_module=__name__, + category=TableCategory.METADATA, + table=_SyntheticTagTable.__table__, # ty: ignore[invalid-argument-type] + primary_key_columns=tuple(_SyntheticTagTable.__table__.primary_key.columns), # ty: ignore[unresolved-attribute] + ) + monkeypatch.setattr(cli_schema_tables, "collect_missing_tables", lambda *a, **k: [fake_table]) + + try: + results = create_missing_tables(engine, resolved=resolved) + result_by_name = {result.table_name: result for result in results} + assert result_by_name["synthetic_tag_table"].status == "created" + assert sa.inspect(engine).has_table("synthetic_tag_table") + finally: + Base.registry._dispose_cls(_SyntheticTagTable) + Base.metadata.remove(_SyntheticTagTable.__table__) # ty: ignore[invalid-argument-type] diff --git a/tests/test_data_summary.py b/tests/test_data_summary.py index 0b4605d..82ace8f 100644 --- a/tests/test_data_summary.py +++ b/tests/test_data_summary.py @@ -4,45 +4,38 @@ from omop_alchemy.maintenance.cli_schema import collect_data_summary -def _engine(tmp_path): - return sa.create_engine(f"sqlite:///{tmp_path / 'data_summary.db'}", future=True) - - -def test_collect_data_summary_can_include_missing_tables(tmp_path): +def test_collect_data_summary_can_include_missing_tables(fresh_engine): """Test collect data summary can include missing tables.""" - engine = _engine(tmp_path) - results = collect_data_summary(engine, existing_only=False) + results = collect_data_summary(fresh_engine, existing_only=False) assert results assert any(result.exists is False for result in results) -def test_collect_data_summary_reports_row_counts(tmp_path): +def test_collect_data_summary_reports_row_counts(fresh_engine): """Test collect data summary reports row counts.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) - with engine.begin() as connection: + with fresh_engine.begin() as connection: connection.execute( sa.text("INSERT INTO location (location_id) VALUES (1)") ) results = { result.table_name: result - for result in collect_data_summary(engine, vocabulary_included=True) + for result in collect_data_summary(fresh_engine, vocabulary_included=True) } assert results["location"].exists is True assert results["location"].row_count == 1 -def test_collect_data_summary_excludes_vocabulary_by_default(tmp_path): +def test_collect_data_summary_excludes_vocabulary_by_default(fresh_engine): """Test collect data summary excludes vocabulary by default.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) table_names = { result.table_name - for result in collect_data_summary(engine) + for result in collect_data_summary(fresh_engine) } assert "person" in table_names assert "concept" not in table_names diff --git a/tests/test_episode_attachment_queries.py b/tests/test_episode_attachment_queries.py index 0285fab..f987abd 100644 --- a/tests/test_episode_attachment_queries.py +++ b/tests/test_episode_attachment_queries.py @@ -352,7 +352,7 @@ def test_diagnostics_explain_person_mismatches_and_fallback_outcomes(session): "session_fixture", [ "session", - pytest.param("pg_session", marks=pytest.mark.requires_database("test_cdm_db")), + pytest.param("pg_session", marks=pytest.mark.db_dialect), ], ) @pytest.mark.parametrize("copies", [(2, 1), (1, 2), (2, 2)]) @@ -455,7 +455,6 @@ def test_attachment_builder_accepts_a_supported_event_model(): ) -@pytest.mark.requires_database("test_cdm_db") def test_postgresql_executes_collision_and_stable_tie_contracts(pg_session): unlinked = EventCase( identity=ClinicalEventIdentity("procedure_occurrence", 8), diff --git a/tests/test_episodes_basic.py b/tests/test_episodes_basic.py index 9bf3ed8..0cb1b0a 100644 --- a/tests/test_episodes_basic.py +++ b/tests/test_episodes_basic.py @@ -1,3 +1,4 @@ +from oa_configurator import Role from omop_alchemy.cdm.base import ModifierFieldConcepts from omop_alchemy.cdm.model.structural import ( EpisodeView, @@ -19,6 +20,7 @@ class DiagnosticEpisode(ResolvedEpisodeEventMixin, EpisodeView): """Non-oncology episode view used to verify the generic extension point.""" __tablename__ = "episode" + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 12264a3..3dcaeeb 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -1,4 +1,3 @@ -import sqlalchemy as sa import pytest from typer.testing import CliRunner @@ -12,37 +11,31 @@ collect_foreign_key_trigger_status, manage_foreign_key_triggers, ) -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Role, StackConfig runner = CliRunner() -def _engine(tmp_path): - return sa.create_engine(f"sqlite:///{tmp_path / 'foreign_keys.db'}", future=True) - - -def test_collect_fk_info_finds_participating_tables(tmp_path): +def test_collect_fk_info_finds_participating_tables(fresh_engine): """Test _collect_fk_info finds participating tables.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) targets = { target.table_name: target - for target in _collect_fk_info(engine) + for target in _collect_fk_info(fresh_engine) } assert "person" in targets assert targets["person"].incoming_constraint_count > 0 -def test_manage_foreign_key_triggers_supports_dry_run(tmp_path): +def test_manage_foreign_key_triggers_supports_dry_run(fresh_engine): """Test manage foreign key triggers supports dry run.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) with pytest.raises(RuntimeError) as exc_info: manage_foreign_key_triggers( - engine, + fresh_engine, enable=False, dry_run=True, ) @@ -50,24 +43,22 @@ def test_manage_foreign_key_triggers_supports_dry_run(tmp_path): assert "not supported by the SQLite backend" in str(exc_info.value) -def test_collect_foreign_key_trigger_status_is_safe_on_sqlite(tmp_path): +def test_collect_foreign_key_trigger_status_is_safe_on_sqlite(fresh_engine): """Test collect foreign key trigger status is safe on sqlite.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) with pytest.raises(RuntimeError) as exc_info: - collect_foreign_key_trigger_status(engine) + collect_foreign_key_trigger_status(fresh_engine) assert "not supported by the SQLite backend" in str(exc_info.value) -def test_validate_foreign_key_constraints_is_safe_on_sqlite(tmp_path): +def test_validate_foreign_key_constraints_is_safe_on_sqlite(fresh_engine): """Test validate foreign key constraints is safe on sqlite.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) with pytest.raises(RuntimeError) as exc_info: - validate_foreign_key_constraints(engine) + validate_foreign_key_constraints(fresh_engine) assert "not supported by the SQLite backend" in str(exc_info.value) @@ -77,7 +68,7 @@ def test_disable_foreign_keys_cli_fails_gracefully_for_sqlite(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -109,10 +100,10 @@ def name(self) -> str: def dialect(self) -> str: return "postgresql" - def analyze_table(self, conn, table_name, db_schema, *, vacuum=False) -> None: + def analyze_table(self, conn, table_name, *, vacuum=False, schema_tag=None) -> None: pass - def toggle_fk_triggers(self, conn, table_name, db_schema, *, enable: bool) -> None: + def toggle_fk_triggers(self, conn, table_name, *, enable: bool, schema_tag=None) -> None: action = "ENABLE" if enable else "DISABLE" conn.exec_driver_sql(f"ALTER TABLE {table_name} {action} TRIGGER ALL") @@ -147,10 +138,11 @@ def begin(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_fk_info", - lambda engine, *, db_schema=None, vocabulary_included=False: [ + lambda engine, *, vocabulary_included=False: [ type("Target", (), { "table_name": "person", "category": "clinical", + "schema_tag": Role.PRIMARY.value, "model_name": "Person", "model_module": "omop_alchemy.cdm.model.clinical.person", "outgoing_constraint_count": 1, @@ -159,6 +151,7 @@ def begin(self): type("Target", (), { "table_name": "visit_occurrence", "category": "health_system", + "schema_tag": Role.PRIMARY.value, "model_name": "VisitOccurrence", "model_module": "omop_alchemy.cdm.model.health_system.visit_occurrence", "outgoing_constraint_count": 2, @@ -168,7 +161,7 @@ def begin(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_strict_validation_failures", - lambda connection, backend, *, db_schema=None, vocabulary_included=False: { + lambda connection, backend, *, vocabulary_included=False: { "visit_occurrence": [ ForeignKeyConstraintViolation( source_table_name="visit_occurrence", @@ -217,10 +210,11 @@ def begin(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_fk_info", - lambda engine, *, db_schema=None, vocabulary_included=False: [ + lambda engine, *, vocabulary_included=False: [ type("Target", (), { "table_name": "person", "category": "clinical", + "schema_tag": Role.PRIMARY.value, "model_name": "Person", "model_module": "omop_alchemy.cdm.model.clinical.person", "outgoing_constraint_count": 1, @@ -230,7 +224,7 @@ def begin(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_strict_validation_failures", - lambda connection, backend, *, db_schema=None, vocabulary_included=False: {}, + lambda connection, backend, *, vocabulary_included=False: {}, ) results = manage_foreign_key_triggers( @@ -251,7 +245,7 @@ def test_enable_foreign_keys_strict_cli_invokes_strict_management(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -314,10 +308,11 @@ def connect(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_fk_info", - lambda engine, *, db_schema=None, vocabulary_included=False: [ + lambda engine, *, vocabulary_included=False: [ type("Target", (), { "table_name": "person", "category": "clinical", + "schema_tag": Role.PRIMARY.value, "model_name": "Person", "model_module": "omop_alchemy.cdm.model.clinical.person", "outgoing_constraint_count": 1, @@ -326,6 +321,7 @@ def connect(self): type("Target", (), { "table_name": "visit_occurrence", "category": "health_system", + "schema_tag": Role.PRIMARY.value, "model_name": "VisitOccurrence", "model_module": "omop_alchemy.cdm.model.health_system.visit_occurrence", "outgoing_constraint_count": 2, @@ -335,7 +331,7 @@ def connect(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_strict_validation_failures", - lambda connection, backend, *, db_schema=None, vocabulary_included=False: { + lambda connection, backend, *, vocabulary_included=False: { "visit_occurrence": [ ForeignKeyConstraintViolation( source_table_name="visit_occurrence", @@ -363,7 +359,7 @@ def test_foreign_keys_validate_cli_invokes_validation(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -395,6 +391,7 @@ def fake_validate_foreign_key_constraints( ForeignKeyValidationResult( table_name="visit_occurrence", category=TableCategory.HEALTH_SYSTEM, + schema_tag=Role.PRIMARY.value, outgoing_constraint_count=2, incoming_constraint_count=0, violating_constraint_count=1, diff --git a/tests/test_fulltext.py b/tests/test_fulltext.py index 9b08ef8..4036e24 100644 --- a/tests/test_fulltext.py +++ b/tests/test_fulltext.py @@ -1,7 +1,9 @@ import sqlalchemy as sa import pytest +from sqlalchemy.dialects import postgresql from typer.testing import CliRunner -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, SCHEMA_TRANSLATE_MAP_KEY, StackConfig +from oa_configurator import Role as SchemaRole from omop_alchemy.backends import ( CONCEPT_NAME_TSVECTOR_COLUMN, @@ -35,9 +37,20 @@ def __init__(self, rowcount: int | None = None): class _FakeConnection: - def __init__(self, *, rowcount: int = 7): + def __init__(self, *, rowcount: int = 7, db_schema: str | None = "public"): self.calls: list[tuple[str, str, dict[str, object] | None]] = [] self.rowcount = rowcount + self.dialect = postgresql.dialect() + self._db_schema = db_schema + + def get_execution_options(self) -> dict[str, object]: + return { + SCHEMA_TRANSLATE_MAP_KEY: { + SchemaRole.PRIMARY.value: self._db_schema, + SchemaRole.VOCAB.value: self._db_schema, + SchemaRole.RESULTS.value: self._db_schema, + } + } def exec_driver_sql( self, @@ -70,8 +83,8 @@ def __exit__(self, exc_type, exc, tb) -> bool: class _FakeEngine: dialect = _FakeDialect() - def __init__(self, *, rowcount: int = 7): - self.connection = _FakeConnection(rowcount=rowcount) + def __init__(self, *, rowcount: int = 7, db_schema: str | None = "public"): + self.connection = _FakeConnection(rowcount=rowcount, db_schema=db_schema) def begin(self) -> _FakeBegin: return _FakeBegin(self.connection) @@ -113,7 +126,6 @@ def test_install_fulltext_columns_builds_postgresql_ddl_and_registers_metadata() results = install_fulltext_columns( engine, # type: ignore[arg-type] - db_schema="public", create_indexes=True, fastupdate=True, ) @@ -122,7 +134,7 @@ def test_install_fulltext_columns_builds_postgresql_ddl_and_registers_metadata() assert all(result.status == "applied" for result in results) statements = [call[1] for call in engine.connection.calls] assert any( - 'ALTER TABLE "public"."concept" ADD COLUMN IF NOT EXISTS concept_name_tsvector tsvector' in statement + 'ALTER TABLE public.concept ADD COLUMN IF NOT EXISTS concept_name_tsvector tsvector' in statement for statement in statements ) assert any( @@ -140,15 +152,14 @@ def test_populate_fulltext_columns_issues_update_with_regconfig_and_row_counts() results = populate_fulltext_columns( engine, # type: ignore[arg-type] - db_schema="public", regconfig="simple", ) assert all(result.status == "applied" for result in results) assert [result.row_count for result in results] == [11, 11] execute_calls = [call for call in engine.connection.calls if call[0] == "execute"] - assert any('UPDATE "public"."concept"' in call[1] for call in execute_calls) - assert any("CAST(:regconfig AS regconfig)" in call[1] for call in execute_calls) + assert any('UPDATE public.concept' in call[1] for call in execute_calls) + assert any("CAST(:regconfig AS REGCONFIG)" in call[1] for call in execute_calls) assert all(call[2] == {"regconfig": "simple"} for call in execute_calls) _postgres.unregister_fulltext_metadata() @@ -160,16 +171,15 @@ def test_drop_fulltext_columns_drops_schema_objects_and_unregisters_metadata(): results = drop_fulltext_columns( engine, # type: ignore[arg-type] - db_schema="public", drop_indexes=True, ) assert [result.action for result in results] == [FullTextAction.DROP, FullTextAction.DROP] assert all(result.status == "applied" for result in results) statements = [call[1] for call in engine.connection.calls] - assert any('DROP INDEX IF EXISTS "public"."idx_gin_concept_name_tsvector"' in statement for statement in statements) + assert any('DROP INDEX IF EXISTS public.idx_gin_concept_name_tsvector' in statement for statement in statements) assert any( - 'ALTER TABLE "public"."concept" DROP COLUMN IF EXISTS concept_name_tsvector' in statement + 'ALTER TABLE public.concept DROP COLUMN IF EXISTS concept_name_tsvector' in statement for statement in statements ) assert CONCEPT_NAME_TSVECTOR_COLUMN not in Concept.__table__.c @@ -183,9 +193,9 @@ def test_drop_fulltext_columns_drops_schema_objects_and_unregisters_metadata(): "drop_fulltext_columns", ], ) -def test_fulltext_management_requires_postgresql(tmp_path, fn_name): +def test_fulltext_management_requires_postgresql(fresh_engine, fn_name): """Fulltext management APIs reject non-PostgreSQL engines.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'fulltext.db'}", future=True) + engine = fresh_engine fn = { "install_fulltext_columns": install_fulltext_columns, "populate_fulltext_columns": populate_fulltext_columns, @@ -204,8 +214,12 @@ def test_fulltext_install_cli_passes_options(monkeypatch): calls: dict[str, object] = {} cfg = StackConfig.for_session( - connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="public")}, + connections={ + "db": ConnectionConfig( + dialect="postgresql+psycopg", host="localhost", database_name="db" + ) + }, + databases={"cdm_db": CDMDatabaseConfig(connection="db", cdm_schema="public")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -215,13 +229,12 @@ def test_fulltext_install_cli_passes_options(monkeypatch): def fake_install_fulltext_columns( engine: object, *, - db_schema: str | None = None, create_indexes: bool = True, fastupdate: bool = False, dry_run: bool = False, + resolved: object = None, ): calls["engine"] = engine - calls["db_schema"] = db_schema calls["create_indexes"] = create_indexes calls["fastupdate"] = fastupdate calls["dry_run"] = dry_run @@ -254,7 +267,6 @@ def fake_install_fulltext_columns( ) assert result.exit_code == 0 - assert calls["db_schema"] == "public" assert calls["fastupdate"] is True assert calls["dry_run"] is True assert "fulltext install" in result.stdout diff --git a/tests/test_indexes.py b/tests/test_indexes.py index db357b2..9796dae 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -1,13 +1,16 @@ import pytest import sqlalchemy as sa +from pydantic import ValidationError from typer.testing import CliRunner -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Role, StackConfig, qualified +from oa_configurator.testing import DIALECT_PARAMS from omop_alchemy.backends.sqlite import SQLiteBackend from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY, omop_index_name from omop_alchemy.maintenance.cli import app from omop_alchemy.maintenance.cli_schema import create_missing_tables -from omop_alchemy.maintenance._cli_utils import ReservedSchema, Status, reject_reserved_schema +from omop_alchemy.config import MAINTENANCE_SCHEMA +from omop_alchemy.maintenance._cli_utils import Status from omop_alchemy.maintenance.ui import render_index_summary from omop_alchemy.maintenance.cli_indexes import ( IndexManagementResult, @@ -37,16 +40,51 @@ CONCEPT_SYNONYM_NAME_LOWER_INDEX = "ix_concept_synonym_concept_synonym_name_lower" -def _fresh_engine(tmp_path): - db_path = tmp_path / "indexes.db" - engine = sa.create_engine(f"sqlite:///{db_path}", future=True) +@pytest.fixture(params=DIALECT_PARAMS) +def indexed_engine(request): + """Every OMOP table created, on both a real Postgres backend and + SQLite: index bookkeeping/capture/clustering is dialect-portable logic, + not SQLite-specific. Only the postgresql param ever requests + pg_session, so the sqlite param never needs a database. + + pg_session's own isolation gives every test a fresh, uniquely-named + CDM schema (see pg_engine), but MAINTENANCE_SCHEMA is a separate, + fixed, shared schema bookkeeping tables always live in regardless of + that per-test uniqueness, so it is never reset by pg_session at all. + The bookkeeping table is dropped here explicitly instead. Otherwise + both its rows *and its very existence* stay visible to every later + test, indefinitely, including tests that specifically assert it hasn't + been created yet. + """ + if request.param == "postgresql": + engine = request.getfixturevalue("pg_session").get_bind() + bookkeeping_schema = get_bookkeeping_schema(engine) + inspector = sa.inspect(engine) + if inspector.has_table(_DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema): + table_ref = qualified(engine, _DROPPED_INDEXES_TABLE_NAME, physical_schema=bookkeeping_schema) + with engine.begin() as connection: + connection.exec_driver_sql(f"DROP TABLE {table_ref}") + else: + engine = request.getfixturevalue("fresh_engine") create_missing_tables(engine) return engine -def test_collect_index_targets_excludes_vocabulary_by_default(tmp_path): +@pytest.fixture +def sqlite_indexed_engine(fresh_engine): + """fresh_engine with every OMOP table already created. + + For the handful of tests that monkeypatch SQLiteBackend's own methods, + or assert SQLite-specific reflection limitations directly, since those + are dialect-specific by construction, not candidates for indexed_engine. + """ + create_missing_tables(fresh_engine) + return fresh_engine + + +def test_collect_index_targets_excludes_vocabulary_by_default(indexed_engine): """Test collect index targets excludes vocabulary by default.""" - engine = _fresh_engine(tmp_path) + engine = indexed_engine targets = { (target.table_name, target.index_name) for target in collect_index_targets(engine) @@ -59,7 +97,7 @@ def test_collect_index_targets_excludes_vocabulary_by_default(tmp_path): @pytest.mark.filterwarnings( "ignore:Skipped unsupported reflection of expression-based index:sqlalchemy.exc.SAWarning" ) -def test_collect_index_targets_can_include_vocabulary(tmp_path): +def test_collect_index_targets_can_include_vocabulary(indexed_engine): """Test collect index targets can include vocabulary. Notes @@ -71,7 +109,7 @@ def test_collect_index_targets_can_include_vocabulary(tmp_path): for coverage of the indexes themselves. """ - engine = _fresh_engine(tmp_path) + engine = indexed_engine targets = { (target.table_name, target.index_name) for target in collect_index_targets(engine, vocabulary_included=True) @@ -99,25 +137,18 @@ def test_orm_index_metadata_carries_cluster_configuration(): def test_schema_metadata_indexes_keys_match_unadjusted_indexes(): - """Test schema metadata indexes keys match unadjusted indexes. - - manage_indexes() looks up schema-adjusted indexes using names taken from - the original, unadjusted ORM tables. If a column's index name is resolved - implicitly (e.g. via `index=True` rather than an explicit omop_index()), - SQLAlchemy's naming convention embeds the schema into the generated name, - so the schema-adjusted copy gets a different name and the lookup misses. - """ + """Every table/index pair from ORM metadata is present in the result.""" tables = select_omop_tables(vocabulary_included=True) - indexes = _schema_metadata_indexes(tables, db_schema="public") + indexes = _schema_metadata_indexes(tables) for table in tables: for index in table.table.indexes: assert (table.table_name, str(index.name)) in indexes -def test_manage_indexes_disable_and_enable_on_sqlite(tmp_path): +def test_manage_indexes_disable_and_enable_on_sqlite(sqlite_indexed_engine): """Test manage indexes disable and enable on sqlite.""" - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine inspector = sa.inspect(engine) before = { @@ -162,17 +193,17 @@ def test_manage_indexes_disable_and_enable_on_sqlite(tmp_path): assert PERSON_GENDER_INDEX in after_enable -def test_manage_indexes_enable_analyzes_tables_with_new_indexes(tmp_path, monkeypatch): +def test_manage_indexes_enable_analyzes_tables_with_new_indexes(sqlite_indexed_engine, monkeypatch): """Test manage indexes enable analyzes tables with new indexes.""" - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine manage_indexes(engine, enable=False) analyzed_tables: list[str] = [] original_analyze = SQLiteBackend.analyze_table - def recording_analyze(self, conn, table_name, db_schema, *, vacuum=False): + def recording_analyze(self, conn, table_name, *, vacuum=False, schema_tag=Role.PRIMARY.value): analyzed_tables.append(table_name) - return original_analyze(self, conn, table_name, db_schema, vacuum=vacuum) + return original_analyze(self, conn, table_name, vacuum=vacuum, schema_tag=schema_tag) monkeypatch.setattr(SQLiteBackend, "analyze_table", recording_analyze) @@ -181,15 +212,15 @@ def recording_analyze(self, conn, table_name, db_schema, *, vacuum=False): assert "person" in analyzed_tables -def test_manage_indexes_enable_skips_analyze_when_nothing_created(tmp_path, monkeypatch): +def test_manage_indexes_enable_skips_analyze_when_nothing_created(sqlite_indexed_engine, monkeypatch): """Test manage indexes enable skips analyze when nothing created.""" - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine analyzed_tables: list[str] = [] monkeypatch.setattr( SQLiteBackend, "analyze_table", - lambda self, conn, table_name, db_schema, *, vacuum=False: analyzed_tables.append(table_name), + lambda self, conn, table_name, *, vacuum=False: analyzed_tables.append(table_name), ) # All ORM-defined indexes already exist on a freshly created schema, so @@ -202,7 +233,7 @@ def test_manage_indexes_enable_skips_analyze_when_nothing_created(tmp_path, monk @pytest.mark.filterwarnings( "ignore:Skipped unsupported reflection of expression-based index:sqlalchemy.exc.SAWarning" ) -def test_manage_indexes_enable_is_idempotent_for_expression_indexes(tmp_path): +def test_manage_indexes_enable_is_idempotent_for_expression_indexes(sqlite_indexed_engine): """Test manage indexes enable is idempotent for expression indexes. SQLite cannot reflect expression-based indexes (e.g. lower(concept_name)), @@ -211,7 +242,7 @@ def test_manage_indexes_enable_is_idempotent_for_expression_indexes(tmp_path): resulting duplicate-create attempt and must report it as skipped rather than falsely claiming the index was (re)created. """ - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine for _ in range(2): results = manage_indexes(engine, enable=True, vocabulary_included=True) @@ -232,7 +263,7 @@ def test_manage_indexes_enable_is_idempotent_for_expression_indexes(tmp_path): @pytest.mark.filterwarnings( "ignore:Skipped unsupported reflection of expression-based index:sqlalchemy.exc.SAWarning" ) -def test_manage_indexes_disable_drops_expression_indexes_on_sqlite(tmp_path): +def test_manage_indexes_disable_drops_expression_indexes_on_sqlite(sqlite_indexed_engine): """Test manage indexes disable drops expression indexes on sqlite. SQLite can't reflect expression-based indexes, so `disable` must not gate @@ -242,7 +273,7 @@ def test_manage_indexes_disable_drops_expression_indexes_on_sqlite(tmp_path): the same way. The index must actually be removed, and a result row must always be reported. """ - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine def _index_exists(name: str) -> bool: with engine.connect() as connection: @@ -275,9 +306,9 @@ def _index_exists(name: str) -> bool: @pytest.mark.filterwarnings( "ignore:Skipped unsupported reflection of expression-based index:sqlalchemy.exc.SAWarning" ) -def test_manage_indexes_disable_is_idempotent_on_sqlite(tmp_path): +def test_manage_indexes_disable_is_idempotent_on_sqlite(indexed_engine): """A second disable run should report already-absent indexes as skipped.""" - engine = _fresh_engine(tmp_path) + engine = indexed_engine first_results = manage_indexes(engine, enable=False, vocabulary_included=True) second_results = manage_indexes(engine, enable=False, vocabulary_included=True) @@ -301,7 +332,7 @@ def test_manage_indexes_disable_is_idempotent_on_sqlite(tmp_path): assert "already absent" in result.detail -def test_manage_indexes_enable_clusters_then_analyzes(tmp_path, monkeypatch): +def test_manage_indexes_enable_clusters_then_analyzes(sqlite_indexed_engine, monkeypatch): """Test manage indexes enable clusters then analyzes, even with nothing created. `indexes enable --cluster` must ANALYZE a table whenever it was clustered, @@ -310,14 +341,14 @@ def test_manage_indexes_enable_clusters_then_analyzes(tmp_path, monkeypatch): clustering so planner stats reflect the final physical layout -- matching the standalone `indexes cluster` command's order. """ - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine calls: list[str] = [] - def fake_cluster_table(self, conn, table_name, index_name, db_schema): + def fake_cluster_table(self, conn, table_name, index_name, *, schema_tag=Role.PRIMARY.value): calls.append(f"cluster:{table_name}") - def fake_analyze_table(self, conn, table_name, db_schema, *, vacuum=False): + def fake_analyze_table(self, conn, table_name, *, vacuum=False, schema_tag=Role.PRIMARY.value): calls.append(f"analyze:{table_name}") monkeypatch.setattr(SQLiteBackend, "cluster_table", fake_cluster_table) @@ -339,7 +370,7 @@ def test_disable_indexes_cli_invokes_management(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -353,6 +384,7 @@ def fake_manage_indexes( db_schema: str | None = None, vocabulary_included: bool = False, dry_run: bool = False, + resolved: object = None, ) -> list[IndexManagementResult]: calls["engine"] = engine calls["enable"] = enable @@ -364,6 +396,7 @@ def fake_manage_indexes( operation="index", table_name="person", category=TableCategory.CLINICAL, + schema_tag=Role.PRIMARY.value, index_name=PERSON_GENDER_INDEX, column_names=("gender_concept_id",), unique=False, @@ -403,7 +436,7 @@ def test_enable_indexes_cli_no_cluster_flag_passes_through(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -418,6 +451,7 @@ def fake_manage_indexes( vocabulary_included: bool = False, dry_run: bool = False, cluster: bool = True, + resolved: object = None, ) -> list[IndexManagementResult]: calls["enable"] = enable calls["vocabulary_included"] = vocabulary_included @@ -428,6 +462,7 @@ def fake_manage_indexes( operation="index", table_name="person", category=TableCategory.CLINICAL, + schema_tag=Role.PRIMARY.value, index_name=PERSON_GENDER_INDEX, column_names=("gender_concept_id",), unique=False, @@ -499,6 +534,20 @@ def test_is_plain_index_false_for_non_btree_index(): assert _is_plain_index(_NON_BTREE) is False +def test_is_plain_index_false_for_sqlite_partial_index(): + """A SQLite partial index, reflected with a sqlite_where dialect + option, must not be classified as plain. Treating it as plain would + make it look safe to drop and recreate from a bare column list, which + would silently lose its WHERE predicate.""" + sqlite_partial = { + "name": "idx_partial_sqlite", + "column_names": ["gender_concept_id"], + "unique": False, + "dialect_options": {"sqlite_where": "gender_concept_id IS NOT NULL"}, + } + assert _is_plain_index(sqlite_partial) is False + + def test_find_equivalent_index_is_order_sensitive(): existing = [{"name": "idx_ab", "column_names": ["b", "a"], "unique": False}] assert _find_equivalent_index(existing, ("a", "b"), False) is None @@ -529,19 +578,35 @@ def test_describe_shape_conflict_mentions_reason(): assert "non-btree access method 'gin'" in _describe_shape_conflict(_NON_BTREE) -# ── Reserved schema guard ──────────────────────────────────────────────────────── - - -def test_reject_reserved_schema_rejects_staging_and_maintenance(): - with pytest.raises(RuntimeError): - reject_reserved_schema(ReservedSchema.STAGING.value) - with pytest.raises(RuntimeError): - reject_reserved_schema(ReservedSchema.MAINTENANCE.value) +def test_describe_shape_conflict_mentions_reason_for_sqlite_dialect_options(): + """The conflict reason for a SQLite index must name the actual cause + (a WHERE predicate or a non-btree access method), read from its + sqlite_where/sqlite_using dialect options, not just Postgres's.""" + sqlite_partial = { + "dialect_options": {"sqlite_where": "gender_concept_id IS NOT NULL"}, + } + sqlite_non_btree = { + "dialect_options": {"sqlite_using": "gin"}, + } + assert "partial WHERE predicate" in _describe_shape_conflict(sqlite_partial) + assert "non-btree access method 'gin'" in _describe_shape_conflict(sqlite_non_btree) -def test_reject_reserved_schema_allows_ordinary_schema(): - reject_reserved_schema("public") - reject_reserved_schema(None) +# ── Reserved schema guard ──────────────────────────────────────────────────────── +# The check itself lives in oa_configurator (resolved via CDMDatabaseConfig.resolve(), +# see oa-configurator's own test_resolver.py::TestReservedSchemaCollision). This +# confirms OMOP_Alchemy's own MAINTENANCE_SCHEMA registration (_cli_utils.py, +# module import time) is actually picked up by it. + + +def test_resolving_cdm_database_with_maintenance_schema_name_raises(): + with pytest.raises(ValidationError, match=f"{MAINTENANCE_SCHEMA!r}.*omop_alchemy"): + StackConfig.for_session( + connections={"c": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, + databases={ + "default": CDMDatabaseConfig(connection="c", cdm_schema=MAINTENANCE_SCHEMA) + }, + ) # ── Foreign-named equivalent index reconciliation ──────────────────────────────── @@ -570,8 +635,8 @@ def _person_gender_result(results: list[IndexManagementResult]) -> IndexManageme return matches[0] -def test_collect_index_targets_reports_foreign_named_equivalent_index(tmp_path): - engine = _fresh_engine(tmp_path) +def test_collect_index_targets_reports_foreign_named_equivalent_index(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") targets = { @@ -583,8 +648,8 @@ def test_collect_index_targets_reports_foreign_named_equivalent_index(tmp_path): assert ("person", PERSON_GENDER_INDEX) not in targets -def test_manage_indexes_enable_skips_creation_when_foreign_equivalent_exists(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_enable_skips_creation_when_foreign_equivalent_exists(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") results = manage_indexes(engine, enable=True, cluster=False) @@ -600,8 +665,8 @@ def test_manage_indexes_enable_skips_creation_when_foreign_equivalent_exists(tmp assert PERSON_GENDER_INDEX not in index_names -def test_manage_indexes_disable_captures_and_drops_foreign_equivalent_index(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_disable_captures_and_drops_foreign_equivalent_index(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") results = manage_indexes(engine, enable=False) @@ -620,8 +685,8 @@ def test_manage_indexes_disable_captures_and_drops_foreign_equivalent_index(tmp_ assert bookkeeping[0]["index_name"] == "idx_gender" -def test_manage_indexes_enable_restores_captured_foreign_index(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_enable_restores_captured_foreign_index(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") manage_indexes(engine, enable=False) @@ -643,8 +708,8 @@ def test_manage_indexes_enable_restores_captured_foreign_index(tmp_path): assert _dropped_indexes_rows(engine) == [] -def test_manage_indexes_disable_enable_round_trip_is_idempotent_with_capture(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_disable_enable_round_trip_is_idempotent_with_capture(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") for _ in range(2): @@ -657,8 +722,8 @@ def test_manage_indexes_disable_enable_round_trip_is_idempotent_with_capture(tmp assert _dropped_indexes_rows(engine) == [] -def test_manage_indexes_dry_run_previews_foreign_equivalent_without_mutating(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_dry_run_previews_foreign_equivalent_without_mutating(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") results = manage_indexes(engine, enable=True, dry_run=True, cluster=False) @@ -669,24 +734,24 @@ def test_manage_indexes_dry_run_previews_foreign_equivalent_without_mutating(tmp inspector = sa.inspect(engine) assert not inspector.has_table( - _DROPPED_INDEXES_TABLE_NAME, schema=get_bookkeeping_schema(SQLiteBackend()) + _DROPPED_INDEXES_TABLE_NAME, schema=get_bookkeeping_schema(engine) ) -def test_bookkeeping_table_not_created_when_nothing_to_capture(tmp_path): - engine = _fresh_engine(tmp_path) +def test_bookkeeping_table_not_created_when_nothing_to_capture(indexed_engine): + engine = indexed_engine manage_indexes(engine, enable=False) manage_indexes(engine, enable=True, cluster=False) inspector = sa.inspect(engine) assert not inspector.has_table( - _DROPPED_INDEXES_TABLE_NAME, schema=get_bookkeeping_schema(SQLiteBackend()) + _DROPPED_INDEXES_TABLE_NAME, schema=get_bookkeeping_schema(engine) ) -def test_manage_indexes_enable_cluster_uses_restored_physical_name(tmp_path, monkeypatch): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_enable_cluster_uses_restored_physical_name(sqlite_indexed_engine, monkeypatch): + engine = sqlite_indexed_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {EPISODE_PERSON_INDEX}") @@ -698,14 +763,14 @@ def test_manage_indexes_enable_cluster_uses_restored_physical_name(tmp_path, mon calls: list[tuple[str, str]] = [] - def fake_cluster_table(self, conn, table_name, index_name, db_schema): + def fake_cluster_table(self, conn, table_name, index_name, *, schema_tag=Role.PRIMARY.value): calls.append((table_name, index_name)) monkeypatch.setattr(SQLiteBackend, "cluster_table", fake_cluster_table) monkeypatch.setattr( SQLiteBackend, "analyze_table", - lambda self, conn, table_name, db_schema, *, vacuum=False: None, + lambda self, conn, table_name, *, vacuum=False, schema_tag=Role.PRIMARY.value: None, ) manage_indexes(engine, enable=True, cluster=True) @@ -714,9 +779,9 @@ def fake_cluster_table(self, conn, table_name, index_name, db_schema): def test_manage_indexes_disable_warns_and_leaves_unsupported_foreign_index_in_place( - tmp_path, monkeypatch + indexed_engine, monkeypatch ): - engine = _fresh_engine(tmp_path) + engine = indexed_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") @@ -756,13 +821,14 @@ def fake_get_indexes(self, table_name, schema=None, **kw): def _dropped_indexes_rows(engine: sa.Engine) -> list[dict[str, object]]: - bookkeeping_schema = get_bookkeeping_schema(SQLiteBackend()) + bookkeeping_schema = get_bookkeeping_schema(engine) inspector = sa.inspect(engine) if not inspector.has_table(_DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema): return [] + table_ref = qualified(engine, _DROPPED_INDEXES_TABLE_NAME, physical_schema=bookkeeping_schema) with engine.connect() as connection: rows = connection.exec_driver_sql( - f"SELECT table_name, index_name FROM {_DROPPED_INDEXES_TABLE_NAME}" + f"SELECT table_name, index_name FROM {table_ref}" ).mappings().all() return [dict(row) for row in rows] @@ -781,6 +847,7 @@ def _warning_result() -> IndexManagementResult: operation="index", table_name="person", category=TableCategory.CLINICAL, + schema_tag=Role.PRIMARY.value, index_name="idx_gender_partial", column_names=("gender_concept_id",), unique=False, @@ -802,6 +869,7 @@ def test_render_index_summary_omits_warnings_row_when_none(): operation="index", table_name="person", category=TableCategory.CLINICAL, + schema_tag=Role.PRIMARY.value, index_name=PERSON_GENDER_INDEX, column_names=("gender_concept_id",), unique=False, @@ -833,12 +901,12 @@ def test_is_plain_index_false_for_constraint_backed_index(): assert "constraint" in _describe_shape_conflict(constraint_backed) -def test_manage_indexes_disable_second_run_without_enable_degrades_to_warning(tmp_path): +def test_manage_indexes_disable_second_run_without_enable_degrades_to_warning(indexed_engine): """A second disable run, without an intervening enable, that finds a different foreign index than the one already captured must not crash. It must leave the second index in place and report a warning, since the bookkeeping table can only track one pending capture per table/column-set.""" - engine = _fresh_engine(tmp_path) + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender_v1") first = manage_indexes(engine, enable=False) @@ -863,21 +931,20 @@ def test_manage_indexes_disable_second_run_without_enable_degrades_to_warning(tm assert restored.index_name == "idx_gender_v1" -def test_record_captured_index_scopes_by_db_schema(tmp_path): +def test_record_captured_index_scopes_by_db_schema(indexed_engine): """Two different schemas capturing an equivalent foreign index for a same-named table/column-set must not collide. Each capture is independent and neither should be rejected by the other's bookkeeping row.""" - engine = _fresh_engine(tmp_path) - backend = SQLiteBackend() + engine = indexed_engine with engine.begin() as connection: captured_a = _record_captured_index( - connection, backend, + connection, table_name="person", db_schema="site_a", index_name="idx_a", column_names=("gender_concept_id",), unique=False, ) captured_b = _record_captured_index( - connection, backend, + connection, table_name="person", db_schema="site_b", index_name="idx_b", column_names=("gender_concept_id",), unique=False, ) @@ -886,11 +953,10 @@ def test_record_captured_index_scopes_by_db_schema(tmp_path): assert captured_b is True # Capturing again for the *same* schema, with a different foreign name, must - # still be rejected (this is the case test_manage_indexes_disable_second_run_ - # without_enable_degrades_to_warning covers end-to-end). + # still be rejected. with engine.begin() as connection: captured_a_again = _record_captured_index( - connection, backend, + connection, table_name="person", db_schema="site_a", index_name="idx_a_v2", column_names=("gender_concept_id",), unique=False, ) @@ -929,14 +995,9 @@ def test_resolve_physical_cluster_name_ignores_uniqueness_for_pk_based_target(): def test_vocabulary_domain_concept_class_relationship_cluster_on_primary_key(): - """vocabulary/domain/concept_class/relationship previously declared a - redundant secondary index as their cluster target (same column as their own - primary key), inconsistently with person/location/care_site/provider/concept - which cluster directly on the primary key's own index. Normalized onto the - latter: Alchemy never creates that redundant index itself, and index - reconciliation is what recognizes a database that has the OHDSI-standard - duplicate (see _resolve_physical_cluster_name_falls_back_to_equivalent) - as an equivalent cluster target.""" + """vocabulary/domain/concept_class/relationship must cluster directly on + their own primary key's index, the same as person/location/care_site/ + provider/concept do, not on a separate, redundant same-column index.""" tables = {table.table_name: table for table in collect_maintenance_tables()} for table_name, pk_column in ( ("vocabulary", "vocabulary_id"), diff --git a/tests/test_inspect_functional_index_probe.py b/tests/test_inspect_functional_index_probe.py new file mode 100644 index 0000000..81055e7 --- /dev/null +++ b/tests/test_inspect_functional_index_probe.py @@ -0,0 +1,25 @@ +import pytest +import sqlalchemy as sa +from oa_configurator.testing import isolated_test_schema +from omop_alchemy.maintenance.cli_schema import create_missing_tables + +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + +def test_inspect_functional_index(pg_db, pg_engine): + with isolated_test_schema(pg_engine, prefix="funcidx_probe") as schema: + import dataclasses + resolved = dataclasses.replace( + pg_db.resolved, schema_name=schema, vocab_schema=schema, results_schema=schema + ) + engine = pg_engine.execution_options( + schema_translate_map={"primary": schema, "vocab": schema, "results": schema} + ) + create_missing_tables(engine, vocabulary_included=True, resolved=resolved) + inspector = sa.inspect(engine) + with engine.connect() as conn: + inspector = sa.inspect(conn) + for idx in inspector.get_indexes("concept", schema=schema): + if idx["name"] == "ix_concept_concept_name_lower": + print("FULL DICT:", idx) + for k, v in idx.items(): + print(f" {k!r}: {v!r}") diff --git a/tests/test_load_vocab.py b/tests/test_load_vocab.py index 05ecda8..db2bc3e 100644 --- a/tests/test_load_vocab.py +++ b/tests/test_load_vocab.py @@ -1,6 +1,5 @@ import pytest from orm_loader.helpers import bootstrap -import sqlalchemy as sa from sqlalchemy.orm import sessionmaker from omop_alchemy.cdm.model.vocabulary import ( @@ -14,21 +13,15 @@ from tests.conftest import ATHENA_LOAD_ORDER, _ATHENA_FIXTURE_DATA, _write_fixture_csv -@pytest.fixture(scope="session") -def connection(): +@pytest.fixture +def connection(fresh_engine): """ In-memory SQLite database for tests. """ - engine = sa.create_engine( - "sqlite+pysqlite:///:memory:", - future=True, - ) - - connection = engine.connect() + connection = fresh_engine.connect() bootstrap(connection, create=True) # type: ignore[arg-type] yield connection connection.close() - engine.dispose() @pytest.fixture @@ -46,15 +39,15 @@ def db_session(connection): session.close() -@pytest.fixture(scope="session") -def athena_vocab(connection, tmp_path_factory): +@pytest.fixture +def athena_vocab(connection, tmp_path): """ Load the minimal Athena vocabulary fixture using the real ORM CSV loader. Writes in-memory fixture data to a temp directory so no static CSV files on disk are required. """ - base_path: Path = tmp_path_factory.mktemp("athena_vocab") + base_path: Path = tmp_path Session = sessionmaker(bind=connection, future=True) session = Session() diff --git a/tests/test_load_vocab_postgres.py b/tests/test_load_vocab_postgres.py index df905a6..50c0976 100644 --- a/tests/test_load_vocab_postgres.py +++ b/tests/test_load_vocab_postgres.py @@ -1,9 +1,12 @@ """ PostgreSQL integration tests for OMOP_Alchemy vocabulary loading. -These tests require a dedicated ``test_cdm_db`` PostgreSQL resource configured -with ``test_only = true``. Then run: - pytest -m requires_database +These tests require a running PostgreSQL container. Start one with: + docker compose -f tests/docker-compose.yaml up -d + +Excluded from the default `pytest` invocation (addopts = "-m 'not +db_dialect'", see oa_configurator.testing). Run explicitly: + pytest -m postgresql """ from pathlib import Path @@ -11,6 +14,7 @@ import pytest import sqlalchemy as sa +from oa_configurator import Role from omop_alchemy.backends.postgres import PostgresBackend from omop_alchemy.cdm.model.vocabulary import Concept from omop_alchemy.maintenance.cli_vocab import ( @@ -19,6 +23,8 @@ ) from tests.conftest import _ATHENA_FIXTURE_DATA, _write_fixture_csv +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + def _copy_fixture_source(base_dir: Path) -> Path: """Write the shared in-memory Athena fixture set into an isolated per-test source dir.""" @@ -73,7 +79,6 @@ def _make_concept_source( # --------------------------------------------------------------------------- -@pytest.mark.requires_database("test_cdm_db") def test_end_to_end_vocab_load_on_postgres(pg_session, pg_engine, tmp_path): """load_vocab_source() completes end-to-end on real Postgres via orm-loader>=0.4.0.""" source_path = _copy_fixture_source(tmp_path) @@ -87,7 +92,6 @@ def test_end_to_end_vocab_load_on_postgres(pg_session, pg_engine, tmp_path): assert count == 7 -@pytest.mark.requires_database("test_cdm_db") def test_default_quote_mode_preserves_literal_quotes_on_postgres( pg_session, pg_engine, tmp_path ): @@ -131,7 +135,6 @@ def test_default_quote_mode_preserves_literal_quotes_on_postgres( assert concept_name == quoted_name -@pytest.mark.requires_database("test_cdm_db") def test_explicit_csv_quote_mode_strips_quotes_on_postgres( pg_session, pg_engine, tmp_path ): @@ -171,7 +174,6 @@ def test_explicit_csv_quote_mode_strips_quotes_on_postgres( assert concept_name == long_name -@pytest.mark.requires_database("test_cdm_db") def test_load_vocab_model_csv_on_postgres(pg_session, tmp_path): """ _load_vocab_model_csv loads data correctly on a real PostgreSQL session. @@ -195,7 +197,6 @@ def test_load_vocab_model_csv_on_postgres(pg_session, tmp_path): assert count == 7 -@pytest.mark.requires_database("test_cdm_db") def test_replace_strategy_overwrites_matching_and_preserves_absent_rows( pg_session, pg_engine, @@ -233,7 +234,6 @@ def test_replace_strategy_overwrites_matching_and_preserves_absent_rows( assert names[source_absent_id] == "preserved" -@pytest.mark.requires_database("test_cdm_db") def test_upsert_strategy_is_non_destructive(pg_session, pg_engine, tmp_path): """merge_strategy='upsert' preserves existing rows on second load with same PKs.""" concept_id = 99998 @@ -256,11 +256,16 @@ def test_upsert_strategy_is_non_destructive(pg_session, pg_engine, tmp_path): ) -@pytest.mark.requires_database("test_cdm_db") def test_db_schema_search_path_on_postgres(pg_engine, tmp_path): """ load_vocab_source with db_schema creates vocabulary tables in the requested PostgreSQL schema and loads data into them correctly. + + schema_translate_map, not db_schema alone, is what actually routes + ORM-managed table creation: a real deployment sets it once, at engine + construction (ResolvedCDMDatabase.create_engine()), not per call. This + scopes it here the same way, matching orm-loader's own + test_schema_translate_map.py regression test. """ schema = "VocabTest" source_path = _copy_fixture_source(tmp_path) @@ -271,9 +276,16 @@ def test_db_schema_search_path_on_postgres(pg_engine, tmp_path): conn.execute(sa.text(f"CREATE SCHEMA {quoted_schema}")) conn.commit() + # A single-schema deployment: vocab/results fall back to the same + # schema as everything else, matching ResolvedCDMDatabase's own default + # fallback behaviour when vocab_schema/results_schema aren't configured. + scoped_engine = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema, "vocab": schema, "results": schema} + ) + try: report = load_vocab_source( - pg_engine, + scoped_engine, source_path=source_path, db_schema=schema, ) @@ -296,7 +308,6 @@ def test_db_schema_search_path_on_postgres(pg_engine, tmp_path): conn.commit() -@pytest.mark.requires_database("test_cdm_db") def test_postgres_catalog_queries_accept_explicit_schema(pg_engine): """Schema-qualified catalog checks must bind cleanly with psycopg/PostgreSQL.""" backend = PostgresBackend() @@ -305,12 +316,10 @@ def test_postgres_catalog_queries_accept_explicit_schema(pg_engine): disabled, enabled = backend.get_fk_trigger_counts( connection, "concept", - "public", ) clustered_index = backend.get_clustered_index_name( connection, "concept", - "public", ) assert disabled >= 0 diff --git a/tests/test_load_vocab_source.py b/tests/test_load_vocab_source.py index 01d55a6..eb83e83 100644 --- a/tests/test_load_vocab_source.py +++ b/tests/test_load_vocab_source.py @@ -2,7 +2,7 @@ import pytest import sqlalchemy as sa -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Role, StackConfig from sqlalchemy.orm import sessionmaker from typer.testing import CliRunner @@ -59,13 +59,12 @@ def _write_csv_with_size(source_path: Path, table_name: str, size_bytes: int) -> def test_load_vocab_source_on_sqlite_creates_tables_and_reports_loaded_results( + fresh_engine, monkeypatch, tmp_path, ): """Test load vocab source on sqlite creates tables and reports loaded results.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) loaded_tables: list[tuple[str, str, str, Path]] = [] @@ -114,11 +113,9 @@ def fake_load_vocab_model_csv( assert inspector.has_table("concept") -def test_load_vocab_source_requires_full_required_athena_fixture(tmp_path): +def test_load_vocab_source_requires_full_required_athena_fixture(fresh_engine, tmp_path): """Test load vocab source requires full required athena fixture.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_missing_required.db'}", future=True - ) + engine = fresh_engine # Build a source with only a subset of required models to trigger the missing-files error. partial_source = tmp_path / "partial_athena" @@ -146,11 +143,9 @@ def test_drug_strength_model_matches_athena_vocabulary_shape(): assert "end_datetime" not in column_names -def test_load_vocab_source_dry_run_does_not_create_tables(tmp_path): +def test_load_vocab_source_dry_run_does_not_create_tables(fresh_engine, tmp_path): """Test load vocab source dry run does not create tables.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_dry_run.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) report = load_vocab_source( @@ -182,7 +177,7 @@ def test_load_vocab_source_cli_uses_configured_athena_source(monkeypatch, tmp_pa connections={ "db": ConnectionConfig(dialect="sqlite", database_name=":memory:") }, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, tools={OmopAlchemyConfig.tool_name: {"athena_source_path": str(athena_dir)}}, ) @@ -194,6 +189,8 @@ def test_load_vocab_source_cli_uses_configured_athena_source(monkeypatch, tmp_pa def fake_load_vocab_source( engine: object, *, + vocab_engine: object = None, # noqa: ARG001 + vocab_schema: str | None = None, # noqa: ARG001 source_path: str | Path, tables: list[str] | None = None, # noqa: ARG001 db_schema: str | None = None, @@ -204,6 +201,7 @@ def fake_load_vocab_source( bulk_mode: bool = True, merge_batch_size: int = 1_000_000, progress_callback=None, + resolved: object = None, # noqa: ARG001 ): calls["source_path"] = str(source_path) calls["dry_run"] = dry_run @@ -241,11 +239,9 @@ def fake_load_vocab_source( assert "load-vocab-source" in result.stdout -def test_load_vocab_model_csv_passes_quote_mode(monkeypatch, tmp_path): +def test_load_vocab_model_csv_passes_quote_mode(fresh_engine, monkeypatch, tmp_path): """Test load vocab model csv passes quote mode.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_quote_mode.db'}", future=True - ) + engine = fresh_engine class FakeModel: __tablename__ = "concept" @@ -294,11 +290,9 @@ def fake_load_csv( assert calls["quote_mode"] == "literal" -def test_load_vocab_source_loads_in_fk_dependency_order(monkeypatch, tmp_path): +def test_load_vocab_source_loads_in_fk_dependency_order(fresh_engine, monkeypatch, tmp_path): """Tables must be loaded in REQUIRED_VOCAB_MODELS order to respect FK dependencies.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_order.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) # Give domain a tiny file and concept_class a large one — if size-sorting were still in place @@ -335,11 +329,9 @@ def fake_load_vocab_model_csv( assert loaded_order[: len(expected_order)] == expected_order -def test_load_vocab_source_reports_weighted_progress(monkeypatch, tmp_path): +def test_load_vocab_source_reports_weighted_progress(fresh_engine, monkeypatch, tmp_path): """Test load vocab source reports weighted progress.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_progress.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) _write_csv_with_size(source_path, "domain", 10) @@ -379,11 +371,9 @@ def fake_load_vocab_model_csv( assert percents == sorted(percents) -def test_load_vocab_source_wraps_failed_table_load(monkeypatch, tmp_path): +def test_load_vocab_source_wraps_failed_table_load(fresh_engine, monkeypatch, tmp_path): """Test load vocab source wraps failed table load.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_error.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) def fake_load_vocab_model_csv( @@ -423,11 +413,9 @@ def fake_load_vocab_model_csv( assert "value too long for type character varying(255)" in message -def test_load_vocab_model_csv_retries_missing_staging_table(monkeypatch, tmp_path): +def test_load_vocab_model_csv_retries_missing_staging_table(fresh_engine, monkeypatch, tmp_path): """Test load vocab model csv retries missing staging table.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_retry.db'}", future=True - ) + engine = fresh_engine class FakeModel: __tablename__ = "drug_strength" @@ -494,7 +482,7 @@ def test_load_vocab_source_cli_surfaces_database_error_detail(monkeypatch): connections={ "db": ConnectionConfig(dialect="sqlite", database_name=":memory:") }, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -527,11 +515,9 @@ def fail_load_vocab_source(*args, **kwargs): assert "value too long for type character varying(255)" in result.stdout -def test_load_vocab_source_defaults_to_by_delimiter_quote_mode(monkeypatch, tmp_path): +def test_load_vocab_source_defaults_to_by_delimiter_quote_mode(fresh_engine, monkeypatch, tmp_path): """Tab-delimited Athena quotes are literal data unless explicitly overridden.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'quote_mode_default.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) received_quote_modes: list[str] = [] @@ -565,20 +551,18 @@ def fake_load_vocab_model_csv( assert "csv" not in received_quote_modes -def test_load_vocab_source_tables_unknown_name_raises_runtime_error(tmp_path): +def test_load_vocab_source_tables_unknown_name_raises_runtime_error(fresh_engine, tmp_path): """Unknown table name in tables= is rejected before any DB connection.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'tables_unknown.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) with pytest.raises(RuntimeError, match="Unknown vocabulary table"): load_vocab_source(engine, source_path=source_path, tables=["not_a_table"]) -def test_load_vocab_source_tables_single_loads_only_that_table(monkeypatch, tmp_path): +def test_load_vocab_source_tables_single_loads_only_that_table(fresh_engine, monkeypatch, tmp_path): """tables=['concept'] loads only concept and skips every other table.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'tables_single.db'}", future=True) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) loaded_tables: list[str] = [] @@ -609,11 +593,9 @@ def fake_load_vocab_model_csv( assert result_names == {"concept"} -def test_load_vocab_source_tables_multiple_loads_exactly_those(monkeypatch, tmp_path): +def test_load_vocab_source_tables_multiple_loads_exactly_those(fresh_engine, monkeypatch, tmp_path): """tables=['concept', 'vocabulary'] loads exactly those two tables.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'tables_multiple.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) loaded_tables: list[str] = [] @@ -642,11 +624,9 @@ def fake_load_vocab_model_csv( assert set(loaded_tables) == {"concept", "vocabulary"} -def test_load_vocab_source_tables_skips_required_files_preflight(tmp_path): +def test_load_vocab_source_tables_skips_required_files_preflight(fresh_engine, tmp_path): """tables= skips the all-required-files gate even when most CSVs are absent.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'tables_preflight.db'}", future=True - ) + engine = fresh_engine # Only concept.csv present — would fail the all-required-files check without tables=. source_path = tmp_path / "sparse" @@ -663,11 +643,9 @@ def test_load_vocab_source_tables_skips_required_files_preflight(tmp_path): assert result_names == {"concept"} -def test_load_vocab_source_tables_missing_csv_raises_runtime_error(tmp_path): +def test_load_vocab_source_tables_missing_csv_raises_runtime_error(fresh_engine, tmp_path): """Explicitly named table whose CSV is absent raises RuntimeError, not a silent skip.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'tables_missing_csv.db'}", future=True - ) + engine = fresh_engine source_path = tmp_path / "empty_source" source_path.mkdir() @@ -677,13 +655,11 @@ def test_load_vocab_source_tables_missing_csv_raises_runtime_error(tmp_path): load_vocab_source(engine, source_path=source_path, tables=["concept"]) -def test_load_vocab_source_bulk_mode_surfaces_index_warnings(monkeypatch, tmp_path): +def test_load_vocab_source_bulk_mode_surfaces_index_warnings(fresh_engine, monkeypatch, tmp_path): """A foreign index that manage_indexes(enable=False) leaves in place (status=warning) during the bulk-mode disable step must be surfaced on the returned report, not silently discarded -- this is the only call site that inspects those results.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_bulk.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) # Force the bulk-mode gate (which requires a PostgreSQL engine) without needing @@ -723,6 +699,7 @@ def fake_manage_indexes(engine, *, enable, **kwargs): operation="index", table_name="concept", category=TableCategory.VOCABULARY, + schema_tag=Role.VOCAB.value, index_name="idx_concept_partial", column_names=("domain_id",), unique=False, @@ -738,6 +715,7 @@ def fake_manage_indexes(engine, *, enable, **kwargs): operation="index", table_name="concept", category=TableCategory.VOCABULARY, + schema_tag=Role.VOCAB.value, index_name="ix_concept_domain_id", column_names=("domain_id",), unique=False, @@ -816,3 +794,12 @@ def test_render_vocab_index_warnings_lists_messages_when_present(): summary_text = summary_buffer.getvalue() assert "Index warnings" in summary_text assert "1" in summary_text + + +def test_sequence_reset_gate_matches_find_sequence_name_capability(fresh_engine, pg_engine): + """The sequence-reset gate must match find_sequence_name support: + False for SQLite, True for Postgres.""" + from omop_alchemy.backends import backend_supports, resolve_backend + + assert backend_supports(resolve_backend(fresh_engine), "find_sequence_name") is False + assert backend_supports(resolve_backend(pg_engine), "find_sequence_name") is True diff --git a/tests/test_modifier_projections.py b/tests/test_modifier_projections.py index 4a6d215..c620349 100644 --- a/tests/test_modifier_projections.py +++ b/tests/test_modifier_projections.py @@ -6,6 +6,7 @@ import sqlalchemy as sa import sqlalchemy.orm as so from datetime import date, datetime +from oa_configurator import Role from sqlalchemy.dialects import postgresql, sqlite from omop_alchemy.cdm.base import ( @@ -271,12 +272,21 @@ def test_a_source_naming_a_missing_link_column_is_rejected(): @pytest.mark.parametrize("source_model", [MeasurementView, ObservationView]) -def test_modifier_source_subclass_projects_its_own_event_metadata(source_model): +def test_modifier_source_subclass_projects_its_own_event_metadata( + source_model, fresh_engine +): original = clinical_event_model_spec(source_model) specialized = type( f"{source_model.__name__}WithOverriddenEventMetadata", (source_model,), { + # Must match source_model's own schema, or SQLAlchemy silently + # builds a second, unlinked Table object (same footgun __table_args__ + # fixes on every *View class: ModifierTargetMixin's __abstract__ = True + # leaks through inherited attribute lookup for any subclass that + # doesn't redeclare its own table identity). + "__tablename__": source_model.__tablename__, + "__table_args__": {"schema": Role.PRIMARY.value}, "__event_id_col__": "projected_id", "projected_id": so.column_property( getattr(source_model, original.event_id_column) + 100 @@ -292,7 +302,7 @@ def test_modifier_source_subclass_projects_its_own_event_metadata(source_model): assert spec.event_date_column == "projected_date" assert spec.event_datetime_column == "projected_datetime" - engine = sa.create_engine("sqlite://") + engine = fresh_engine Base.metadata.create_all(engine, tables=[source_model.__table__]) with engine.begin() as connection: connection.execute( @@ -328,7 +338,11 @@ def test_modifier_source_subclass_rejects_missing_declared_columns( specialized = type( f"{source_model.__name__}WithMissing{declaration.strip('_')}", (source_model,), - {declaration: "no_such_column"}, + { + "__tablename__": source_model.__tablename__, + "__table_args__": {"schema": Role.PRIMARY.value}, + declaration: "no_such_column", + }, ) with pytest.raises(UnsupportedModifierSourceModelError, match="no_such_column"): canonical_modifier_projection(specialized) @@ -383,9 +397,8 @@ def test_target_resolution_queries_compile_for_supported_models(target): assert "diagnostic_code" in str(queries.diagnostics.compile(dialect=dialect)) -def _seeded_engine(conditions, measurements): - """Build a SQLite database holding the given condition and measurement rows.""" - engine = sa.create_engine("sqlite://") +def _seeded_engine(engine, conditions, measurements): + """Populate fresh_engine with the given condition and measurement rows.""" Base.metadata.create_all( engine, tables=[Condition_Occurrence.__table__, Measurement.__table__] ) @@ -425,9 +438,10 @@ def _diagnostic_codes(engine, target) -> set[tuple[int, str]]: return {(row["modifier_id"], row["diagnostic_code"]) for row in rows} -def test_a_whole_target_table_can_prove_a_dangling_modifier(): +def test_a_whole_target_table_can_prove_a_dangling_modifier(fresh_engine): """Absence from an entire table does mean the target row does not exist.""" engine = _seeded_engine( + fresh_engine, conditions=[_condition(1, 10)], measurements=[_modifier_of(100, 10, 1), _modifier_of(101, 10, 99)], ) @@ -437,7 +451,7 @@ def test_a_whole_target_table_can_prove_a_dangling_modifier(): } -def test_a_filtered_target_never_reports_a_missing_target_event(): +def test_a_filtered_target_never_reports_a_missing_target_event(fresh_engine): """A narrowed target set cannot distinguish a defect from its own filter. Both modifiers below point at conditions that genuinely exist. Reporting @@ -445,6 +459,7 @@ def test_a_filtered_target_never_reports_a_missing_target_event(): data-quality defect for downstream baseline counts. """ engine = _seeded_engine( + fresh_engine, conditions=[_condition(1, 10), _condition(2, 10)], measurements=[_modifier_of(100, 10, 1), _modifier_of(101, 10, 2)], ) @@ -455,9 +470,10 @@ def test_a_filtered_target_never_reports_a_missing_target_event(): assert _diagnostic_codes(engine, narrowed) == set() -def test_a_filtered_target_still_reports_person_mismatch(): +def test_a_filtered_target_still_reports_person_mismatch(fresh_engine): """A mismatch is observed on a row that is present, so it stays provable.""" engine = _seeded_engine( + fresh_engine, conditions=[_condition(1, 10)], measurements=[_modifier_of(100, 11, 1)], ) @@ -517,7 +533,6 @@ def modifier( } -@pytest.mark.requires_database("test_cdm_db") def test_postgresql_executes_modifier_target_validation_contracts(pg_session): """PostgreSQL rejects incomplete and cross-person links before selection.""" diff --git a/tests/test_modifier_selection.py b/tests/test_modifier_selection.py index 486b06a..53f3208 100644 --- a/tests/test_modifier_selection.py +++ b/tests/test_modifier_selection.py @@ -208,7 +208,6 @@ def test_condition_modifier_specs_delegate_to_governed_semantics(): ) -@pytest.mark.requires_database("test_cdm_db") def test_postgresql_executes_modifier_selection_and_stage_policy_contracts(pg_session): """Execute collision, stable-tie, and stage overrides on PostgreSQL.""" expected_by_spec = ( diff --git a/tests/test_oncology_episode.py b/tests/test_oncology_episode.py index 2a03156..80d4fbd 100644 --- a/tests/test_oncology_episode.py +++ b/tests/test_oncology_episode.py @@ -104,9 +104,9 @@ def test_oncology_episode_does_not_expose_generic_drug_episode_interface(): @pytest.fixture -def oncology_session(tmp_path) -> Iterator[so.Session]: +def oncology_session(fresh_engine) -> Iterator[so.Session]: """Committed oncology graph so vocabulary-cache sessions see its closure.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'oncology.db'}") + engine = fresh_engine bootstrap(engine, create=True) rt_concept_id = 9_100_001 diff --git a/tests/test_schema_doctor.py b/tests/test_schema_doctor.py index f7a1e6f..8168c08 100644 --- a/tests/test_schema_doctor.py +++ b/tests/test_schema_doctor.py @@ -5,9 +5,10 @@ def test_doctor_uses_borrowed_engine_without_resolving_config_or_disposing( + fresh_engine, monkeypatch, ) -> None: - engine = sa.create_engine("sqlite://") + engine = fresh_engine disposed_engines: list[sa.engine.Engine] = [] inspected: dict[str, object] = {} original_dispose = sa.engine.Engine.dispose @@ -18,12 +19,10 @@ def fail_config_resolution(): def collect_missing( supplied_engine, *, - db_schema=None, vocabulary_included=True, ): inspected.update( engine=supplied_engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) return [] @@ -43,14 +42,13 @@ def track_dispose(self, *args, **kwargs): vocabulary_included=False, ) - assert report.info.engine_url == "sqlite://" + assert report.info.engine_url == str(engine.url) assert report.info.backend == "sqlite" assert report.info.db_schema == "analytics" assert report.info.resource_name == "manual_cdm" assert report.info.connection_ready is True assert inspected == { "engine": engine, - "db_schema": "analytics", "vocabulary_included": False, } assert engine not in disposed_engines diff --git a/tests/test_schema_provenance_guard.py b/tests/test_schema_provenance_guard.py new file mode 100644 index 0000000..ad31ad1 --- /dev/null +++ b/tests/test_schema_provenance_guard.py @@ -0,0 +1,221 @@ +"""Schema-provenance guard wired into create_missing_tables(), install_fulltext_columns(), +manage_indexes(), and truncate_tables(). + +Live-Postgres regression: proves the guard actually fires and prevents DDL +for a genuinely reconfigured schema, rather than passing by coincidence. + +One case per call site is covered here. The guard's own agree/no-op/test_only +semantics are already exhaustively covered at the primitive level in +oa-configurator's own test suite; what's worth proving per consuming repo +is that each call site is actually wired to it, and a wiring mistake (e.g. +a guard wrapping an empty `pass` instead of the real DDL) would show up here +too. + +pg_engine (unlike pg_db) is a real, committing engine, so every provenance +row this file's tests write is a genuine commit. cleanup_after_test deletes +this test's own schema_provenance rows at teardown (see Phase 10.12 in the +plan). +""" + +from __future__ import annotations + +import dataclasses +import uuid + +import pytest +import sqlalchemy as sa +from oa_configurator import SchemaDriftError, Role +from oa_configurator.domains.resources.sql import SCHEMA_PROVENANCE_SCHEMA, _schema_provenance_table + +from oa_configurator.testing import delete_rows_on_cleanup, isolated_test_schema + +from omop_alchemy.backends import CONCEPT_NAME_TSVECTOR_COLUMN +from omop_alchemy.backends.base import FullTextError +from omop_alchemy.maintenance.cli_fulltext import install_fulltext_columns +from omop_alchemy.maintenance.cli_indexes import manage_indexes +from omop_alchemy.maintenance.cli_schema_tables import create_missing_tables +from omop_alchemy.maintenance.cli_tables import truncate_tables +from omop_alchemy.maintenance.tables import TableCategory + +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + + +def _resolved(pg_db, *, database_name: str, schema: str): + """pg_db.resolved with a unique name (the guard's own key includes it), + all three schemas pointed at schema, and both connection.test_only and + vocab_connection.test_only forced False so the guard doesn't no-op + against pg_db's own test-only marking. A VOCAB-tagged guard reads + vocab_connection specifically (connection_for_role(Role.VOCAB)), so + fixing only connection leaves it silently no-op'd. + """ + return dataclasses.replace( + pg_db.resolved, + name=database_name, + schema_name=schema, + vocab_schema=schema, + results_schema=schema, + connection=dataclasses.replace(pg_db.resolved.connection, test_only=False), + vocab_connection=dataclasses.replace(pg_db.resolved.vocab_connection, test_only=False), + ) + + +def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + create_missing_tables( + engine_a, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_a), + ) + + engine_b = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} + ) + with pytest.raises(SchemaDriftError): + create_missing_tables( + engine_b, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), + ) + + # The guard raises before create_all() runs: schema_b must still be empty. + with engine_b.connect() as conn: + assert sa.inspect(conn).get_table_names(schema=schema_b) == [] + + +def test_install_fulltext_columns_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_ft_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_ft_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + resolved_a = _resolved(pg_db, database_name=database_name, schema=schema_a) + # Guarded create establishes the provenance baseline for schema_a; an + # unguarded create here would leave install_fulltext_columns's own guard + # seeing "tables exist but no record", a false first-time-drift positive. + create_missing_tables(engine_a, vocabulary_included=True, resolved=resolved_a) + install_fulltext_columns(engine_a, resolved=resolved_a) + + engine_b = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} + ) + # install_fulltext_columns wraps every underlying error, including the + # guard's own SchemaDriftError, in its own FullTextError. + with pytest.raises(FullTextError) as exc_info: + install_fulltext_columns( + engine_b, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), + ) + assert isinstance(exc_info.value.__cause__, SchemaDriftError) + + # The guard raises before any ALTER TABLE runs: schema_b's concept table + # must not have picked up the tsvector sidecar column. + with engine_b.connect() as conn: + if sa.inspect(conn).has_table("concept", schema=schema_b): + columns = {c["name"] for c in sa.inspect(conn).get_columns("concept", schema=schema_b)} + assert CONCEPT_NAME_TSVECTOR_COLUMN not in columns + + +def test_manage_indexes_enable_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_idx_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_idx_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + resolved_a = _resolved(pg_db, database_name=database_name, schema=schema_a) + create_missing_tables(engine_a, vocabulary_included=True, resolved=resolved_a) + manage_indexes(engine_a, enable=True, cluster=False, resolved=resolved_a) + + engine_b = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} + ) + with pytest.raises(SchemaDriftError): + manage_indexes( + engine_b, + enable=True, + cluster=False, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), + ) + + +def test_manage_indexes_disable_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): + """Regression for the disable path specifically: manage_indexes(enable=False) + used to build no guard at all, regardless of resolved.""" + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_idxd_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_idxd_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + resolved_a = _resolved(pg_db, database_name=database_name, schema=schema_a) + create_missing_tables(engine_a, vocabulary_included=True, resolved=resolved_a) + manage_indexes(engine_a, enable=False, resolved=resolved_a) + + engine_b = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} + ) + create_missing_tables(engine_b, vocabulary_included=True) + with pytest.raises(SchemaDriftError): + manage_indexes( + engine_b, + enable=False, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), + ) + + +def test_truncate_tables_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_trunc_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_trunc_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + resolved_a = _resolved(pg_db, database_name=database_name, schema=schema_a) + create_missing_tables(engine_a, vocabulary_included=True, resolved=resolved_a) + truncate_tables(engine_a, scope=TableCategory.VOCABULARY, cascade=True, resolved=resolved_a) + + engine_b = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} + ) + create_missing_tables(engine_b, vocabulary_included=True) + with pytest.raises(SchemaDriftError): + truncate_tables( + engine_b, + scope=TableCategory.VOCABULARY, + cascade=True, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), + ) diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index d44d68e..d746d19 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -1,19 +1,87 @@ +import dataclasses +from typing import NamedTuple + +import pytest import sqlalchemy as sa +from oa_configurator import ResolvedCDMDatabase, Role +from oa_configurator import qualified, physical_schema_of, Dialect +from oa_configurator.testing import DIALECT_PARAMS, isolated_test_schema from omop_alchemy.backends.sqlite import SQLiteBackend from omop_alchemy.cdm.base.indexing import omop_index_name +from omop_alchemy.maintenance.cli_indexes import manage_indexes from omop_alchemy.maintenance.cli_schema import create_missing_tables from omop_alchemy.maintenance.cli_schema_reconcile import is_blocking_issue, reconcile_schema +from tests.conftest import resolved_cdm_database_from_engine + PERSON_GENDER_INDEX = omop_index_name("person", "gender_concept_id") EPISODE_PERSON_INDEX = omop_index_name("episode", "person_id") -def _fresh_engine(tmp_path): - db_path = tmp_path / "reconcile.db" - engine = sa.create_engine(f"sqlite:///{db_path}", future=True) +class _ReconcileEngine(NamedTuple): + engine: sa.Engine + resolved: ResolvedCDMDatabase + + +def _sqlite_resolved(engine: sa.Engine) -> ResolvedCDMDatabase: + """A ResolvedCDMDatabase for engine, single-schema (None throughout) to + match fresh_engine's own forced schema_translate_map. SQLite has no real + connection identity to resolve, so building this directly from the + engine's own URL is the whole story, unlike Postgres. + """ + return resolved_cdm_database_from_engine(engine, name="fresh_engine_test") + + +@pytest.fixture(params=DIALECT_PARAMS) +def reconcile_engine(request) -> _ReconcileEngine: + """Every OMOP table created, indexed, and clustered, on both Postgres and SQLite. + + manage_indexes(enable=True) is required on Postgres: create_missing_tables() + alone creates indexes but never physically CLUSTERs them, so a fresh + database would otherwise report false cluster drift. It's a harmless + no-op for clustering on SQLite. + + Notes + ----- + DIALECT_PARAMS marks each param directly, since the postgresql param's + dynamic request.getfixturevalue("pg_schema_session") call is invisible + to pytest's usual fixturenames-based auto-detection. + """ + if request.param == "postgresql": + resolved = request.getfixturevalue("pg_db").resolved + engine = request.getfixturevalue("pg_schema_session").get_bind() + schema = physical_schema_of(engine) + resolved = dataclasses.replace( + resolved, + schema_name=schema, + vocab_schema=schema, + results_schema=schema, + ) + else: + engine = request.getfixturevalue("fresh_engine") + resolved = _sqlite_resolved(engine) create_missing_tables(engine) - return engine + if request.param == Dialect.POSTGRESQL: + manage_indexes(engine, enable=True) + else: + manage_indexes(engine, enable=True) + return _ReconcileEngine(engine, resolved) + + +@pytest.fixture +def fresh_reconcile_engine(fresh_engine) -> _ReconcileEngine: + """fresh_engine with every OMOP table already created. + + SQLite-only, unlike reconcile_engine above: the tests using this + fixture monkeypatch SQLiteBackend.get_clustered_index_name directly to + simulate a physical CLUSTER state, since SQLite has no real clustering + to test against at all (CLUSTER is a genuine Postgres-only physical + operation). Parametrizing these onto Postgres would need a real CLUSTER + call, not a mock swap, so they stay a separate, SQLite-specific fixture. + """ + create_missing_tables(fresh_engine) + return _ReconcileEngine(fresh_engine, _sqlite_resolved(fresh_engine)) def _person_gender_issues(report): @@ -26,22 +94,24 @@ def _person_gender_issues(report): ] -def test_reconcile_schema_reports_no_drift_on_fresh_database(tmp_path): - engine = _fresh_engine(tmp_path) - report = reconcile_schema(engine) +def test_reconcile_schema_reports_no_drift_on_fresh_database(reconcile_engine): + engine, resolved = reconcile_engine + report = reconcile_schema(engine, resolved=resolved) person_result = next(r for r in report.table_results if r.table_name == "person") assert person_result.status == "matched" assert person_result.issue_count == 0 -def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(tmp_path): - engine = _fresh_engine(tmp_path) +def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(reconcile_engine): + engine, resolved = reconcile_engine with engine.begin() as connection: - connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") - connection.exec_driver_sql("CREATE INDEX idx_gender ON person (gender_concept_id)") + connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX, physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))}") + connection.exec_driver_sql( + f"CREATE INDEX idx_gender ON {qualified(connection, 'person', physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))} (gender_concept_id)" + ) - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) issues = _person_gender_issues(report) assert len(issues) == 1 @@ -51,19 +121,138 @@ def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(tmp assert issue.actual == "idx_gender" -def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(tmp_path): - engine = _fresh_engine(tmp_path) +def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(reconcile_engine): + engine, resolved = reconcile_engine with engine.begin() as connection: - connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") - connection.exec_driver_sql("CREATE INDEX idx_gender ON person (gender_concept_id)") + connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX, physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))}") + connection.exec_driver_sql( + f"CREATE INDEX idx_gender ON {qualified(connection, 'person', physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))} (gender_concept_id)" + ) - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) person_result = next(r for r in report.table_results if r.table_name == "person") assert person_result.status == "matched" assert person_result.issue_count == 1 +@pytest.mark.postgresql +@pytest.mark.db_dialect +def test_reconcile_schema_reports_relocated_when_table_found_in_another_schema(pg_db, pg_engine): + """A table missing from its expected schema but physically present under + a different one reports RELOCATED, not a plain MISSING. + """ + with ( + isolated_test_schema(pg_engine, prefix="reconcile_relocated_a") as schema_a, + isolated_test_schema(pg_engine, prefix="reconcile_relocated_b") as schema_b, + ): + resolved = dataclasses.replace( + pg_db.resolved, schema_name=schema_a, vocab_schema=schema_a, results_schema=schema_a + ) + engine = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + # vocabulary_included defaults to True: person's gender_concept_id FK + # targets a vocab table, so excluding vocab here would leave that FK + # unresolved and person itself blocked from creation. + create_missing_tables(engine, resolved=resolved) + with engine.begin() as connection: + connection.exec_driver_sql(f'ALTER TABLE "{schema_a}".person SET SCHEMA "{schema_b}"') + + report = reconcile_schema(engine, resolved=resolved) + + person_result = next(r for r in report.table_results if r.table_name == "person") + assert person_result.status == "relocated" + person_issue = next( + i for i in report.issues if i.table_name == "person" and i.component == "table" + ) + assert person_issue.status == "relocated" + # public may also legitimately carry a person table from an + # unrelated database/test, so assert schema_b is among the + # relocated schemas rather than the only one reported. + assert person_issue.actual is not None + assert schema_b in person_issue.actual.split(", ") + assert is_blocking_issue(person_issue) + + +@pytest.mark.postgresql +@pytest.mark.db_dialect +def test_reconcile_schema_with_resolved_qualifies_each_table_to_its_own_role_schema(pg_db, pg_engine): + """A clinical table, a vocab table, and a results table each live in a + genuinely different physical schema. Passing resolved must compare each + against its own schema, not one blanket db_schema value, or the vocab + and results tables report false drift here. + """ + with ( + isolated_test_schema(pg_engine, prefix="reconcile_three_primary") as primary_schema, + isolated_test_schema(pg_engine, prefix="reconcile_three_vocab") as vocab_schema, + isolated_test_schema(pg_engine, prefix="reconcile_three_results") as results_schema, + ): + resolved = dataclasses.replace( + pg_db.resolved, + schema_name=primary_schema, + vocab_schema=vocab_schema, + results_schema=results_schema, + ) + engine = pg_engine.execution_options( + schema_translate_map={ + Role.PRIMARY.value: primary_schema, "vocab": vocab_schema, "results": results_schema + } + ) + create_missing_tables(engine, resolved=resolved) + manage_indexes(engine, enable=True, vocabulary_included=True) + + report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) + + checked_components = {"table", "column", "primary_key", "foreign_key", "cluster", "index"} + for table_name in ("person", "concept", "observation_period"): + issues = [ + issue + for issue in report.issues + if issue.table_name == table_name and issue.component in checked_components + ] + assert issues == [], (table_name, issues) + + +@pytest.mark.postgresql +@pytest.mark.db_dialect +def test_reconcile_schema_catches_genuine_drift_in_a_functional_index(pg_db, pg_engine): + """concept's ix_concept_concept_name_lower (a functional index, + lower(concept_name)) reports no drift when unchanged, and a real + MISMATCH when its live expression is deliberately altered. Proves the + normalization process compares signatures rather than just silencing the check. + """ + with isolated_test_schema(pg_engine, prefix="reconcile_functional_index") as schema: + resolved = dataclasses.replace( + pg_db.resolved, schema_name=schema, vocab_schema=schema, results_schema=schema + ) + engine = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema, "vocab": schema, "results": schema} + ) + create_missing_tables(engine, resolved=resolved) + + def _index_issues(report): + return [ + issue + for issue in report.issues + if issue.table_name == "concept" and issue.object_name == "ix_concept_concept_name_lower" + ] + + report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) + assert _index_issues(report) == [] + + with engine.begin() as connection: + connection.exec_driver_sql(f'DROP INDEX {qualified(connection, "ix_concept_concept_name_lower", physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))}') + connection.exec_driver_sql( + f'CREATE INDEX ix_concept_concept_name_lower ON {qualified(connection, "concept", physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))} (upper(concept_name))' + ) + + report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) + issues = _index_issues(report) + assert len(issues) == 1 + assert issues[0].status == "mismatch" + + def test_is_blocking_issue_excludes_renamed_only(): from omop_alchemy.maintenance.cli_schema_reconcile import ReconciliationIssue from omop_alchemy.maintenance._cli_utils import Status @@ -83,11 +272,11 @@ def test_is_blocking_issue_excludes_renamed_only(): assert is_blocking_issue(missing) is True -def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_index(tmp_path, monkeypatch): +def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_index(fresh_reconcile_engine, monkeypatch): """A table physically clustered on a foreign-named equivalent of the ORM's cluster index (e.g. captured/restored under its original name by manage_indexes()) must report a 'renamed' cluster issue, not 'mismatch'.""" - engine = _fresh_engine(tmp_path) + engine, resolved = fresh_reconcile_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {EPISODE_PERSON_INDEX}") connection.exec_driver_sql("CREATE INDEX idx_episode_person ON episode (person_id)") @@ -95,12 +284,12 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_inde monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name, db_schema: ( + lambda self, conn, table_name, schema_tag=None: ( "idx_episode_person" if table_name == "episode" else None ), ) - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) episode_result = next(r for r in report.table_results if r.table_name == "episode") cluster_issues = [ issue for issue in report.issues @@ -114,20 +303,20 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_inde assert episode_result.status == "matched" -def test_reconcile_schema_cluster_check_still_reports_real_mismatch(tmp_path, monkeypatch): +def test_reconcile_schema_cluster_check_still_reports_real_mismatch(fresh_reconcile_engine, monkeypatch): """A genuinely different physical cluster state (not just a foreign-named equivalent) must still be reported as drift.""" - engine = _fresh_engine(tmp_path) + engine, resolved = fresh_reconcile_engine monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name, db_schema: ( + lambda self, conn, table_name, schema_tag=None: ( "some_unrelated_index" if table_name == "episode" else None ), ) - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) episode_result = next(r for r in report.table_results if r.table_name == "episode") cluster_issues = [ issue for issue in report.issues @@ -139,29 +328,29 @@ def test_reconcile_schema_cluster_check_still_reports_real_mismatch(tmp_path, mo assert episode_result.status == "drifted" -def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_target(tmp_path, monkeypatch): +def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_target(fresh_reconcile_engine, monkeypatch): """person's cluster target is the primary key's own index ("pk_person"), not a declared secondary index, unlike episode. The official OHDSI CDM DDL always clusters such tables on a separate, non-unique index instead - (e.g. "idx_person_id"): this must still report 'renamed', not 'mismatch', - and the same physical index must not *also* be flagged as an unexpected - plain index -- both are the same latent bug (the cluster target's - equivalence check assuming the PK's own uniqueness applies to whatever - physically serves as the cluster index, and not being shared with the - general index-diffing pass).""" - engine = _fresh_engine(tmp_path) + (e.g. "idx_person_id"). This must still report 'renamed', not + 'mismatch', and the same physical index must not also be flagged as an + unexpected plain index. Both are the same latent bug: the cluster + target's equivalence check assumes the PK's own uniqueness applies to + whatever physically serves as the cluster index, and isn't shared with + the general index-diffing pass.""" + engine, resolved = fresh_reconcile_engine with engine.begin() as connection: connection.exec_driver_sql("CREATE INDEX idx_person_id ON person (person_id)") monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name, db_schema: ( + lambda self, conn, table_name, schema_tag=None: ( "idx_person_id" if table_name == "person" else None ), ) - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) person_result = next(r for r in report.table_results if r.table_name == "person") person_issues = [issue for issue in report.issues if issue.table_name == "person"] cluster_issues = [issue for issue in person_issues if issue.component == "cluster"] @@ -176,3 +365,25 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_tar assert cluster_issues[0].actual == "idx_person_id" assert unexpected_index_issues == [] assert person_result.status == "matched" + + +@pytest.mark.parametrize( + ("a", "b", "should_match"), + [ + pytest.param("lower(x)", "lower(x::text)", True, id="cosmetic-textlike-cast-ignored"), + pytest.param("LOWER(x)", "lower(x::text)", True, id="function-name-case-ignored"), + pytest.param("lower( x )", "lower(x::text)", True, id="incidental-whitespace-ignored"), + pytest.param("sum(x::numeric)", "sum(x::integer)", False, id="meaningful-cast-still-caught"), + pytest.param( + "coalesce(x, 'Unknown')", "coalesce(x, 'unknown')", False, id="literal-case-still-caught" + ), + pytest.param( + "concat_ws(' - ', a, b)", "concat_ws('-', a, b)", False, id="literal-whitespace-still-caught" + ), + ], +) +def test_normalize_index_expression_ignores_cosmetic_noise_but_not_real_drift(a, b, should_match): + from omop_alchemy.backends.postgres import PostgresBackend + + backend = PostgresBackend() + assert (backend.normalize_index_expression(a) == backend.normalize_index_expression(b)) is should_match diff --git a/tests/test_temporal_queries.py b/tests/test_temporal_queries.py index f6d48ba..aafe664 100644 --- a/tests/test_temporal_queries.py +++ b/tests/test_temporal_queries.py @@ -214,7 +214,6 @@ def test_as_of_observation_selection_requires_an_anchor(): ) -@pytest.mark.requires_database("test_cdm_db") def test_postgresql_executes_boundary_and_side_preference_contracts(pg_session): start = sa.literal(date(2026, 1, 15)) end = sa.literal(date(2026, 2, 5)) diff --git a/tests/test_truncate_tables.py b/tests/test_truncate_tables.py index 9982d00..b871bfe 100644 --- a/tests/test_truncate_tables.py +++ b/tests/test_truncate_tables.py @@ -1,8 +1,7 @@ import importlib -import sqlalchemy as sa import pytest from typer.testing import CliRunner -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Role, StackConfig from omop_alchemy.maintenance.cli import app from omop_alchemy.maintenance._cli_utils import Status @@ -14,9 +13,9 @@ truncate_tables_module = importlib.import_module("omop_alchemy.maintenance.cli_tables") -def test_truncate_tables_requires_postgresql(tmp_path): +def test_truncate_tables_requires_postgresql(fresh_engine): """Test truncate tables requires postgresql.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'truncate.db'}", future=True) + engine = fresh_engine with pytest.raises(RuntimeError) as exc_info: truncate_tables(engine, scope=TableCategory.CLINICAL, dry_run=True) @@ -24,9 +23,9 @@ def test_truncate_tables_requires_postgresql(tmp_path): assert "not supported by the SQLite backend" in str(exc_info.value) -def test_truncate_tables_reports_blocking_foreign_key_references(monkeypatch, tmp_path): +def test_truncate_tables_reports_blocking_foreign_key_references(monkeypatch, fresh_engine): """Test truncate tables reports blocking foreign key references.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'truncate_fk.db'}", future=True) + engine = fresh_engine create_missing_tables(engine, vocabulary_included=True) monkeypatch.setattr(truncate_tables_module, "require_backend_support", lambda *args, **kwargs: None) @@ -45,7 +44,7 @@ def test_truncate_tables_cli_requires_confirmation(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -64,7 +63,7 @@ def test_truncate_tables_cli_invokes_management(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -74,15 +73,14 @@ def test_truncate_tables_cli_invokes_management(monkeypatch): def fake_truncate_tables( engine: object, *, - db_schema: str | None = None, scope: TableCategory | None = None, table_names: tuple[str, ...] | None = None, restart_identities: bool = False, cascade: bool = False, dry_run: bool = False, + resolved: object = None, ) -> list[TruncateTableResult]: calls["engine"] = engine - calls["db_schema"] = db_schema calls["scope"] = scope calls["table_names"] = table_names calls["restart_identities"] = restart_identities @@ -92,6 +90,7 @@ def fake_truncate_tables( TruncateTableResult( table_name="person", category=TableCategory.CLINICAL, + schema_tag=Role.PRIMARY.value, row_count=10, status=Status.PLANNED, detail="table would be truncated", diff --git a/tests/test_vocab_results_role_wiring_postgres.py b/tests/test_vocab_results_role_wiring_postgres.py new file mode 100644 index 0000000..2ce0b35 --- /dev/null +++ b/tests/test_vocab_results_role_wiring_postgres.py @@ -0,0 +1,239 @@ +"""Vocab/results-role wiring, exercised on OMOP_Alchemy's own models (Phase 3.2). + +Phase 3's vocab/results-role fix tags every vocabulary table with +``schema=Role.VOCAB.value`` and every derived/results table with +``schema=Role.RESULTS.value``. This is its acceptance test: a single +Postgres connection configured with three genuinely different schema +names for ``schema_name``/``vocab_schema``/``results_schema``, confirming +``create_all`` (now role-aware) places each table in the schema its role +says it belongs in, and that a join across a clinical table's concept_id +FK into the vocab schema compiles and executes correctly in one query -- +the same-connection case, where this is a single eager join, unlike the +split-connection case covered separately in omop-graph's +``test_vocab_split_connection.py``. +""" + +from __future__ import annotations + +import dataclasses +import uuid +from contextlib import ExitStack +from datetime import date +from typing import Iterator, NamedTuple + +import pytest +import sqlalchemy as sa +import sqlalchemy.orm as so + +from oa_configurator import Role +from oa_configurator.testing import isolated_test_schema + +from omop_alchemy.cdm.model.clinical import Observation, Person +from omop_alchemy.cdm.model.derived import Cohort +from omop_alchemy.cdm.model.vocabulary import Concept, Concept_Class, Domain, Vocabulary +from omop_alchemy.maintenance.cli_schema_tables import create_missing_tables + +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + +_TODAY = date(2020, 1, 1) +META_CONCEPT_ID = 0 + + +class _ThreeSchema(NamedTuple): + engine: sa.Engine + clinical_schema: str + vocab_schema: str + results_schema: str + + +@pytest.fixture() +def three_schema(pg_engine: sa.Engine) -> Iterator[_ThreeSchema]: + with ExitStack() as stack: + clinical_schema = stack.enter_context(isolated_test_schema(pg_engine, prefix="phase32_clinical")) + vocab_schema = stack.enter_context(isolated_test_schema(pg_engine, prefix="phase32_vocab")) + results_schema = stack.enter_context(isolated_test_schema(pg_engine, prefix="phase32_results")) + + engine = pg_engine.execution_options( + schema_translate_map={ + Role.PRIMARY.value: clinical_schema, + "vocab": vocab_schema, + "results": results_schema, + } + ) + yield _ThreeSchema( + engine=engine, + clinical_schema=clinical_schema, + vocab_schema=vocab_schema, + results_schema=results_schema, + ) + + +def _bootstrap_vocab(engine: sa.Engine, vocab_schema: str) -> None: + """Insert the minimal, real-FK-checked concept-0 bootstrap that every + concept_id-defaulting column (Person.gender_concept_id and friends, + Observation's required concept FKs) depends on. Domain/Vocabulary/ + Concept_Class/Concept form a genuine insert cycle in Postgres, and + disabling triggers for the load and re-enabling them afterwards is the + same technique production bulk-loads use for this exact reason.""" + vocab_tables = ("domain", "vocabulary", "concept_class", "concept") + with engine.begin() as conn: + for table in vocab_tables: + conn.execute(sa.text(f'ALTER TABLE "{vocab_schema}"."{table}" DISABLE TRIGGER ALL')) + + with so.Session(engine) as session: + session.add_all( + [ + Concept( + concept_id=META_CONCEPT_ID, + concept_name="Meta concept", + domain_id="Metadata", + vocabulary_id="OMOP", + concept_class_id="Metadata", + standard_concept="S", + concept_code="META", + valid_start_date=_TODAY, + valid_end_date=date(2099, 12, 31), + ), + Domain(domain_id="Metadata", domain_name="Metadata", domain_concept_id=META_CONCEPT_ID), + Vocabulary( + vocabulary_id="OMOP", + vocabulary_name="OMOP", + vocabulary_reference="local", + vocabulary_version="test", + vocabulary_concept_id=META_CONCEPT_ID, + ), + Concept_Class( + concept_class_id="Metadata", + concept_class_name="Metadata", + concept_class_concept_id=META_CONCEPT_ID, + ), + ] + ) + session.commit() + + with engine.begin() as conn: + for table in vocab_tables: + conn.execute(sa.text(f'ALTER TABLE "{vocab_schema}"."{table}" ENABLE TRIGGER ALL')) + + +def test_tables_land_in_the_schema_their_role_declares(three_schema: _ThreeSchema) -> None: + create_missing_tables( + three_schema.engine, vocabulary_included=True + ) + + inspector = sa.inspect(three_schema.engine) + assert inspector.has_table("person", schema=three_schema.clinical_schema) + assert inspector.has_table("observation", schema=three_schema.clinical_schema) + assert inspector.has_table("concept", schema=three_schema.vocab_schema) + assert inspector.has_table("domain", schema=three_schema.vocab_schema) + assert inspector.has_table("cohort", schema=three_schema.results_schema) + assert inspector.has_table("observation_period", schema=three_schema.clinical_schema) + + # And not duplicated into the wrong schema. + assert not inspector.has_table("concept", schema=three_schema.clinical_schema) + assert not inspector.has_table("cohort", schema=three_schema.clinical_schema) + + +def test_clinical_to_vocab_join_compiles_and_executes_in_one_query( + three_schema: _ThreeSchema, +) -> None: + create_missing_tables( + three_schema.engine, vocabulary_included=True + ) + _bootstrap_vocab(three_schema.engine, three_schema.vocab_schema) + + with so.Session(three_schema.engine) as session: + session.add( + Person( + person_id=1, + year_of_birth=1990, + gender_concept_id=META_CONCEPT_ID, + race_concept_id=META_CONCEPT_ID, + ethnicity_concept_id=META_CONCEPT_ID, + ) + ) + session.commit() + + session.add( + Observation( + observation_id=1, + person_id=1, + observation_concept_id=META_CONCEPT_ID, + observation_type_concept_id=META_CONCEPT_ID, + observation_date=_TODAY, + ) + ) + session.add( + Cohort( + cohort_definition_id=1, + subject_id=1, + cohort_start_date=_TODAY, + cohort_end_date=_TODAY, + ) + ) + session.commit() + + row = session.execute( + sa.select(Observation.observation_id, Concept.concept_name).join( + Concept, Observation.observation_concept_id == Concept.concept_id + ) + ).one() + assert row.observation_id == 1 + assert row.concept_name == "Meta concept" + + cohort_row = session.execute( + sa.select(Cohort.cohort_definition_id).where(Cohort.subject_id == 1) + ).one() + assert cohort_row.cohort_definition_id == 1 + + +def test_create_missing_tables_creates_vocab_and_results_schemas_on_a_fresh_database( + pg_db, pg_engine: sa.Engine, cleanup_after_test +) -> None: + """create_missing_tables() used to call ensure_schema() only for the + primary schema, so a genuinely fresh database (where vocab/results + schemas don't exist yet either, unlike three_schema's fixture which + pre-creates all three) failed "schema does not exist" for every + vocab/results table. Deliberately doesn't use isolated_test_schema(): + that physically creates the schema up front, which is exactly the step + under test here. + """ + clinical_schema = f"phase32_fresh_clinical_{uuid.uuid4().hex[:8]}" + vocab_schema = f"phase32_fresh_vocab_{uuid.uuid4().hex[:8]}" + results_schema = f"phase32_fresh_results_{uuid.uuid4().hex[:8]}" + + def _drop_schemas() -> None: + with pg_engine.begin() as conn: + for schema in (clinical_schema, vocab_schema, results_schema): + conn.execute(sa.text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')) + + cleanup_after_test(_drop_schemas) + + patched_connection = dataclasses.replace(pg_db.resolved.connection, test_only=False) + resolved = dataclasses.replace( + pg_db.resolved, + name="fresh_schema_test", + schema_name=clinical_schema, + vocab_schema=vocab_schema, + results_schema=results_schema, + connection=patched_connection, + vocab_connection=patched_connection, + ) + engine = pg_engine.execution_options( + schema_translate_map={ + Role.PRIMARY.value: clinical_schema, + "vocab": vocab_schema, + "results": results_schema, + } + ) + + create_missing_tables( + engine, + vocabulary_included=True, + resolved=resolved, + ) + + inspector = sa.inspect(engine) + assert inspector.has_table("person", schema=clinical_schema) + assert inspector.has_table("concept", schema=vocab_schema) + assert inspector.has_table("cohort", schema=results_schema)