From 4504a125080b36e9a1ef2049a4fdd07495a24fe8 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 1 Sep 2026 04:55:08 +0000 Subject: [PATCH 01/32] Reworked config, adapt testing to new oa-config unification, add schemas to tables and cleanup tables not adhering to previously defined standards, adapt CLI to new standard --- omop_alchemy/backends/base.py | 31 --- omop_alchemy/backends/postgres.py | 122 +++++------- omop_alchemy/backends/sqlite.py | 2 - omop_alchemy/cdm/base/__init__.py | 3 +- omop_alchemy/cdm/base/column_helpers.py | 41 +++- omop_alchemy/cdm/base/column_mixins.py | 9 +- .../model/clinical/condition_occurrence.py | 19 +- .../cdm/model/clinical/measurement.py | 17 +- .../cdm/model/clinical/observation.py | 17 +- omop_alchemy/cdm/model/derived/cohort.py | 3 + .../cdm/model/derived/cohort_definition.py | 9 +- .../cdm/model/derived/condition_era.py | 2 + omop_alchemy/cdm/model/derived/dose_era.py | 2 + omop_alchemy/cdm/model/derived/drug_era.py | 2 + .../cdm/model/derived/observation_period.py | 2 + omop_alchemy/cdm/model/metadata/cdm_source.py | 4 +- .../cdm/model/structural/episode_event.py | 4 +- .../cdm/model/structural/fact_relationship.py | 10 +- omop_alchemy/cdm/model/vocabulary/concept.py | 13 +- .../cdm/model/vocabulary/concept_ancestor.py | 7 +- .../cdm/model/vocabulary/concept_class.py | 5 +- .../model/vocabulary/concept_relationship.py | 9 +- .../cdm/model/vocabulary/concept_synonym.py | 7 +- omop_alchemy/cdm/model/vocabulary/domain.py | 5 +- .../cdm/model/vocabulary/drug_strength.py | 14 +- .../cdm/model/vocabulary/relationship.py | 7 +- .../model/vocabulary/source_to_concept_map.py | 7 +- .../cdm/model/vocabulary/vocabulary.py | 5 +- omop_alchemy/config.py | 29 ++- omop_alchemy/maintenance/_cli_utils.py | 48 ++--- omop_alchemy/maintenance/cli_backup.py | 8 +- omop_alchemy/maintenance/cli_foreign_keys.py | 8 +- omop_alchemy/maintenance/cli_fulltext.py | 8 +- omop_alchemy/maintenance/cli_indexes.py | 89 ++++----- omop_alchemy/maintenance/cli_schema.py | 1 + .../maintenance/cli_schema_reconcile.py | 9 +- .../maintenance/cli_schema_summary.py | 5 +- omop_alchemy/maintenance/cli_schema_tables.py | 93 +++++---- omop_alchemy/maintenance/cli_tables.py | 22 +- omop_alchemy/maintenance/cli_vocab.py | 75 +++---- omop_alchemy/maintenance/tables.py | 27 --- pyproject.toml | 2 + tests/conftest.py | 126 ++++++++---- tests/test_analyze_tables.py | 21 +- ...st_backends_non_default_schema_postgres.py | 134 +++++++++++++ tests/test_concept_groups.py | 14 +- tests/test_create_tables.py | 35 ++-- tests/test_data_summary.py | 25 +-- tests/test_foreign_keys.py | 37 ++-- tests/test_fulltext.py | 26 ++- tests/test_indexes.py | 188 ++++++++++-------- tests/test_load_vocab.py | 19 +- tests/test_load_vocab_postgres.py | 32 +-- tests/test_load_vocab_source.py | 89 +++------ tests/test_oncology_episode.py | 4 +- tests/test_schema_doctor.py | 5 +- tests/test_schema_reconcile.py | 78 ++++++-- tests/test_truncate_tables.py | 9 +- ...test_vocab_results_role_wiring_postgres.py | 184 +++++++++++++++++ 59 files changed, 1107 insertions(+), 721 deletions(-) create mode 100644 tests/test_backends_non_default_schema_postgres.py create mode 100644 tests/test_vocab_results_role_wiring_postgres.py diff --git a/omop_alchemy/backends/base.py b/omop_alchemy/backends/base.py index 8db8571..634ba59 100644 --- a/omop_alchemy/backends/base.py +++ b/omop_alchemy/backends/base.py @@ -83,7 +83,6 @@ def toggle_fk_triggers( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, enable: bool, ) -> None: @@ -93,7 +92,6 @@ def get_fk_trigger_counts( self, conn: sa.Connection, table_name: str, - db_schema: str | None, ) -> tuple[int, int]: """Return (disabled_count, enabled_count) for RI triggers on the table.""" raise FeatureNotSupportedError("FK trigger status inspection", self) @@ -105,7 +103,6 @@ def count_fk_violations( referred_table: str, constrained_cols: list[str], referred_cols: list[str], - db_schema: str | None, ) -> int: raise FeatureNotSupportedError("FK constraint violation counting", self) @@ -116,7 +113,6 @@ def cluster_table( conn: sa.Connection, table_name: str, index_name: str, - db_schema: str | None, ) -> None: raise FeatureNotSupportedError("Table clustering", self) @@ -124,7 +120,6 @@ def get_clustered_index_name( self, conn: sa.Connection, table_name: str, - db_schema: str | None, ) -> str | None: raise FeatureNotSupportedError("Cluster index inspection", self) @@ -135,7 +130,6 @@ def analyze_table( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, vacuum: bool = False, ) -> None: ... @@ -144,7 +138,6 @@ def index_exists( self, conn: sa.Connection, index_name: str, - db_schema: str | None, ) -> bool: """Return True when the named index currently exists on the database. @@ -157,7 +150,6 @@ def drop_index_if_exists( self, conn: sa.Connection, index_name: str, - db_schema: str | None, ) -> None: """Drop an index by name without relying on SQLAlchemy's reflection-based checkfirst. @@ -171,7 +163,6 @@ def truncate_table_batch( self, conn: sa.Connection, table_names: list[str], - db_schema: str | None, *, restart_identities: bool, cascade: bool, @@ -185,7 +176,6 @@ def find_sequence_name( conn: sa.Connection, table_name: str, column_name: str, - db_schema: str | None, ) -> str | None: raise FeatureNotSupportedError("Owned sequence lookup", self) @@ -197,22 +187,6 @@ def set_sequence_value( ) -> None: raise FeatureNotSupportedError("Sequence value reset", self) - # ── Schema context ─────────────────────────────────────────────────────── - - def configure_schema_context( - self, - conn: sa.Connection, - db_schema: str | None, - ) -> None: - pass # no-op by default; PostgreSQL overrides with SET search_path - - def ensure_schema( - self, - conn: sa.Connection, - schema: str | None, - ) -> None: - pass # no-op by default; backends that support named schemas override this - # ── Full-text search ───────────────────────────────────────────────────── @property @@ -247,7 +221,6 @@ def install_fulltext_on_table( table_name: str, vector_column_name: str, index_name: str, - db_schema: str | None, create_indexes: bool, fastupdate: bool, ) -> None: @@ -260,7 +233,6 @@ def populate_fulltext_on_table( table_name: str, vector_column_name: str, source_column_name: str, - db_schema: str | None, regconfig: str, ) -> int | None: raise FeatureNotSupportedError("Full-text search", self) @@ -272,7 +244,6 @@ def drop_fulltext_on_table( table_name: str, vector_column_name: str, index_name: str, - db_schema: str | None, drop_indexes: bool, ) -> None: raise FeatureNotSupportedError("Full-text search", self) @@ -284,7 +255,6 @@ def prepare_backup( engine: sa.Engine, output_path: str, backup_format: str, - db_schema: str | None, ) -> tuple[str, list[str], dict[str, str], str]: """Return (tool_path, command, env, database_name). subprocess.run stays in CLI.""" raise FeatureNotSupportedError("Database backup", self) @@ -294,7 +264,6 @@ def prepare_restore( engine: sa.Engine, input_path: str, backup_format: str, - db_schema: str | None, ) -> tuple[str, list[str], dict[str, str], str]: """Return (tool_path, command, env, database_name). subprocess.run stays in CLI.""" raise FeatureNotSupportedError("Database restore", self) diff --git a/omop_alchemy/backends/postgres.py b/omop_alchemy/backends/postgres.py index 19ec50a..f5bc86d 100644 --- a/omop_alchemy/backends/postgres.py +++ b/omop_alchemy/backends/postgres.py @@ -5,24 +5,13 @@ import sqlalchemy as sa -from sqlalchemy.dialects.postgresql import TSVECTOR +from oa_configurator import qualified, schema_of +from sqlalchemy.dialects.postgresql import REGCONFIG, TSVECTOR from sqlalchemy.sql import func from .base import Backend, FullTextTargetConfig -def _qualified(table_name: str, db_schema: str | None) -> str: - if db_schema: - return f'"{db_schema}"."{table_name}"' - return f'"{table_name}"' - - -def _qualified_index(index_name: str, db_schema: str | None) -> str: - if db_schema: - return f'"{db_schema}"."{index_name}"' - return f'"{index_name}"' - - class PostgresBackend(Backend): @property @@ -39,20 +28,18 @@ def toggle_fk_triggers( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, enable: bool, ) -> None: action = "ENABLE" if enable else "DISABLE" conn.exec_driver_sql( - f"ALTER TABLE {_qualified(table_name, db_schema)} {action} TRIGGER ALL" + f"ALTER TABLE {qualified(conn, table_name)} {action} TRIGGER ALL" ) def get_fk_trigger_counts( self, conn: sa.Connection, table_name: str, - db_schema: str | None, ) -> tuple[int, int]: disabled_count, enabled_count = conn.execute( sa.text( @@ -69,7 +56,7 @@ def get_fk_trigger_counts( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": db_schema}, + {"table_name": table_name, "db_schema": schema_of(conn)}, ).one() return int(disabled_count or 0), int(enabled_count or 0) @@ -80,10 +67,9 @@ def count_fk_violations( referred_table: str, constrained_cols: list[str], referred_cols: list[str], - db_schema: str | None, ) -> int: - source = _qualified(source_table, db_schema) - referred = _qualified(referred_table, db_schema) + source = qualified(conn, source_table) + referred = qualified(conn, referred_table) non_null_predicate = " AND ".join( f"src.{col} IS NOT NULL" for col in constrained_cols ) @@ -113,17 +99,15 @@ def cluster_table( conn: sa.Connection, table_name: str, index_name: str, - db_schema: str | None, ) -> None: conn.exec_driver_sql( - f"CLUSTER {_qualified(table_name, db_schema)} USING {index_name}" + f"CLUSTER {qualified(conn, table_name)} USING {index_name}" ) def get_clustered_index_name( self, conn: sa.Connection, table_name: str, - db_schema: str | None, ) -> str | None: result = conn.execute( sa.text( @@ -138,7 +122,7 @@ def get_clustered_index_name( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": db_schema}, + {"table_name": table_name, "db_schema": schema_of(conn)}, ).scalar_one_or_none() return str(result) if result is not None else None @@ -148,20 +132,18 @@ def analyze_table( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, vacuum: bool = False, ) -> None: operation = "VACUUM ANALYZE" if vacuum else "ANALYZE" - conn.exec_driver_sql(f"{operation} {_qualified(table_name, db_schema)}") + conn.exec_driver_sql(f"{operation} {qualified(conn, table_name)}") def index_exists( self, conn: sa.Connection, index_name: str, - db_schema: str | None, ) -> bool: - qualified_index_name = _qualified_index(index_name, db_schema) + qualified_index_name = qualified(conn, index_name) return bool( conn.scalar( sa.select( @@ -170,20 +152,19 @@ def index_exists( ) ) - def drop_index_if_exists(self, conn: sa.Connection, index_name: str, db_schema: str | None) -> None: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {_qualified_index(index_name, db_schema)}") + def drop_index_if_exists(self, conn: sa.Connection, index_name: str) -> None: + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name)}") def truncate_table_batch( self, conn: sa.Connection, table_names: list[str], - db_schema: str | None, *, restart_identities: bool, cascade: bool, ) -> None: sql = "TRUNCATE TABLE " + ", ".join( - _qualified(name, db_schema) for name in table_names + qualified(conn, name) for name in table_names ) if restart_identities: sql += " RESTART IDENTITY" @@ -198,9 +179,8 @@ def find_sequence_name( conn: sa.Connection, table_name: str, column_name: str, - db_schema: str | None, ) -> str | None: - fully_qualified = _qualified(table_name, db_schema) + fully_qualified = qualified(conn, table_name) return conn.execute( sa.text("SELECT pg_get_serial_sequence(:table_name, :column_name)"), {"table_name": fully_qualified, "column_name": column_name}, @@ -217,28 +197,6 @@ def set_sequence_value( {"sequence_name": sequence_name, "value": value}, ) - # ── Schema context ─────────────────────────────────────────────────────── - - def configure_schema_context( - self, - conn: sa.Connection, - db_schema: str | None, - ) -> None: - if db_schema is None: - return - quoted = '"' + db_schema.replace('"', '""') + '"' - conn.exec_driver_sql(f"SET search_path TO {quoted}") - - def ensure_schema( - self, - conn: sa.Connection, - schema: str | None, - ) -> None: - if not schema or schema == "public": - return - quoted = '"' + schema.replace('"', '""') + '"' - conn.exec_driver_sql(f"CREATE SCHEMA IF NOT EXISTS {quoted}") - # ── Full-text search ───────────────────────────────────────────────────── @property @@ -308,20 +266,27 @@ def install_fulltext_on_table( table_name: str, vector_column_name: str, index_name: str, - db_schema: str | None, create_indexes: bool, fastupdate: bool, ) -> None: - qualified_table = _qualified(table_name, db_schema) + qualified_table = qualified(conn, table_name) conn.exec_driver_sql( f"ALTER TABLE {qualified_table} ADD COLUMN IF NOT EXISTS {vector_column_name} tsvector" ) if create_indexes: - conn.exec_driver_sql( - f"CREATE INDEX IF NOT EXISTS {index_name}" - f" ON {qualified_table} USING GIN ({vector_column_name})" - f" WITH (fastupdate = {'on' if fastupdate else 'off'})" + lightweight_table = sa.Table( + table_name, + sa.MetaData(), + sa.Column(vector_column_name, TSVECTOR), + schema=schema_of(conn), + ) + index = sa.Index( + index_name, + lightweight_table.c[vector_column_name], + postgresql_using="gin", + postgresql_with={"fastupdate": "on" if fastupdate else "off"}, ) + conn.execute(sa.schema.CreateIndex(index, if_not_exists=True)) def populate_fulltext_on_table( self, @@ -330,18 +295,24 @@ def populate_fulltext_on_table( table_name: str, vector_column_name: str, source_column_name: str, - db_schema: str | None, regconfig: str, ) -> int | None: - result = conn.execute( - sa.text( - f"UPDATE {_qualified(table_name, db_schema)}" - f" SET {vector_column_name} = to_tsvector(" - f" CAST(:regconfig AS regconfig), coalesce({source_column_name}, '')" - f" )" - ), - {"regconfig": regconfig}, + lightweight_table = sa.table( + table_name, + sa.column(vector_column_name), + sa.column(source_column_name), + schema=schema_of(conn), + ) + source_column = lightweight_table.c[source_column_name] + stmt = lightweight_table.update().values( + **{ + vector_column_name: func.to_tsvector( + sa.cast(sa.bindparam("regconfig"), REGCONFIG), + func.coalesce(source_column, ""), + ) + } ) + result = conn.execute(stmt, {"regconfig": regconfig}) if result.rowcount is None or result.rowcount < 0: return None return int(result.rowcount) @@ -353,13 +324,12 @@ def drop_fulltext_on_table( table_name: str, vector_column_name: str, index_name: str, - db_schema: str | None, drop_indexes: bool, ) -> None: if drop_indexes: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {_qualified_index(index_name, db_schema)}") + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name)}") conn.exec_driver_sql( - f"ALTER TABLE {_qualified(table_name, db_schema)}" + f"ALTER TABLE {qualified(conn, table_name)}" f" DROP COLUMN IF EXISTS {vector_column_name}" ) @@ -370,7 +340,6 @@ def prepare_backup( engine: sa.Engine, output_path: str, backup_format: str, - db_schema: str | None, ) -> tuple[str, list[str], dict[str, str], str]: tool_path = _pg_dump_path() url = engine.url @@ -389,6 +358,7 @@ def prepare_backup( "--no-owner", "--no-privileges", ] + db_schema = schema_of(engine) if db_schema: command.extend(["--schema", db_schema]) env = os.environ.copy() @@ -401,7 +371,6 @@ def prepare_restore( engine: sa.Engine, input_path: str, backup_format: str, - db_schema: str | None, ) -> tuple[str, list[str], dict[str, str], str]: url = engine.url database_name = url.database @@ -410,6 +379,7 @@ def prepare_restore( "Database restore requires a database name in the configured engine URL." ) connection_uri = _libpq_connection_uri(url) + db_schema = schema_of(engine) if backup_format == "custom": tool_path = _pg_restore_path() diff --git a/omop_alchemy/backends/sqlite.py b/omop_alchemy/backends/sqlite.py index d584ca1..b26fedd 100644 --- a/omop_alchemy/backends/sqlite.py +++ b/omop_alchemy/backends/sqlite.py @@ -19,7 +19,6 @@ def index_exists( self, conn: sa.Connection, index_name: str, - db_schema: str | None, ) -> bool: row = conn.exec_driver_sql( "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?", @@ -31,7 +30,6 @@ def analyze_table( self, conn: sa.Connection, table_name: str, - db_schema: str | None, *, vacuum: bool = False, ) -> None: diff --git a/omop_alchemy/cdm/base/__init__.py b/omop_alchemy/cdm/base/__init__.py index fbbe0b1..7c714df 100644 --- a/omop_alchemy/cdm/base/__init__.py +++ b/omop_alchemy/cdm/base/__init__.py @@ -1,6 +1,6 @@ from .cdm_table_base import CDMTableBase from .decorators import cdm_table, MODEL_MODULE_PREFIX -from .column_helpers import required_concept_fk, optional_concept_fk, optional_int, required_int +from .column_helpers import required_concept_fk, optional_concept_fk, role_fk, optional_int, required_int from .column_mixins import ValueMixin, ReferenceTable, DatedEvent, PersonScoped, HealthSystemContext, FactTable from .indexing import merge_table_args, omop_index, omop_primary_key_index_name, omop_table_options from .domain_validation import DomainValidationMixin, DomainRule, ExpectedDomain @@ -17,6 +17,7 @@ "MODEL_MODULE_PREFIX", "required_concept_fk", "optional_concept_fk", + "role_fk", "optional_int", "required_int", "ValueMixin", diff --git a/omop_alchemy/cdm/base/column_helpers.py b/omop_alchemy/cdm/base/column_helpers.py index 84d3480..9409137 100644 --- a/omop_alchemy/cdm/base/column_helpers.py +++ b/omop_alchemy/cdm/base/column_helpers.py @@ -1,5 +1,32 @@ +from typing import Any import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role + + +def role_fk(role: Role, target: str) -> str: + """Schema-qualify an FK target string with a schema role placeholder. + + FK string resolution happens lazily, at mapper-configuration time, so + the target table's own schema role can't be looked up dynamically + without an import-order dependency on whichever file declares it; + role must be given explicitly, matching what the target table + declares in its own __table_args__. + + Parameters + ---------- + role : Role + Schema role the target table is tagged with. + target : str + Unqualified "table.column" FK target string. + + Returns + ------- + str + Schema-qualified FK target string. + """ + return f"{role.value}.{target}" + def required_concept_fk(): """ @@ -15,7 +42,7 @@ def required_concept_fk(): - Must exist - Unknown allowed (concept_id = 0) - Matches CDM Field-Level spec - - foreign key to `concept.concept_id` + - foreign key to `concept.concept_id`, always in the Role.VOCAB schema To index this column, add an explicit `omop_index(...)` to the model's `__table_args__` rather than indexing the column directly — @@ -23,25 +50,28 @@ def required_concept_fk(): """ return so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False, default=0, ) -def optional_concept_fk(): +def optional_concept_fk(**kwargs: Any): """ *optional_concept_fk* Used when a concept reference is genuinely optional. + foreign key to `concept.concept_id`, always in the Role.VOCAB schema. + To index this column, add an explicit `omop_index(...)` to the model's `__table_args__` rather than indexing the column directly — see `omop_alchemy.cdm.base.indexing`. """ return so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=True, + **kwargs, ) def optional_fk(target: str): @@ -50,6 +80,9 @@ def optional_fk(target: str): Optional foreign keys to non-concept tables. + target must already be schema-qualified (see role_fk()) if it points + into a Role-tagged table. + To index this column, add an explicit `omop_index(...)` to the model's `__table_args__` rather than indexing the column directly — see `omop_alchemy.cdm.base.indexing`. diff --git a/omop_alchemy/cdm/base/column_mixins.py b/omop_alchemy/cdm/base/column_mixins.py index eccb9a6..6528e86 100644 --- a/omop_alchemy/cdm/base/column_mixins.py +++ b/omop_alchemy/cdm/base/column_mixins.py @@ -4,6 +4,9 @@ from typing import Optional, Any import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role + +from .column_helpers import role_fk """ @@ -60,7 +63,7 @@ class ValueMixin: This helps when building generic tooling that needs to handle values flexibly but then normalise for analysis. """ value_as_number: so.Mapped[Optional[float]] = so.mapped_column(sa.Float, nullable=True) - value_as_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + value_as_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=True) class DatedEvent: """ @@ -116,7 +119,7 @@ class SourceAttribution: Mixin for *_source_value and *_source_concept_id patterns. """ source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String, nullable=True) - source_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + source_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=True) class UnitConcept: """ @@ -124,4 +127,4 @@ class UnitConcept: Mixin for unit_concept_id. """ - unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=True) diff --git a/omop_alchemy/cdm/model/clinical/condition_occurrence.py b/omop_alchemy/cdm/model/clinical/condition_occurrence.py index e9160ca..b3bff14 100644 --- a/omop_alchemy/cdm/model/clinical/condition_occurrence.py +++ b/omop_alchemy/cdm/model/clinical/condition_occurrence.py @@ -3,18 +3,21 @@ from sqlalchemy.ext.declarative import declared_attr from typing import Optional, TYPE_CHECKING from datetime import date, datetime +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( - PersonScoped, - HealthSystemContext, - FactTable, + PersonScoped, + HealthSystemContext, + FactTable, ReferenceContext, CDMTableBase, - cdm_table, + cdm_table, ModifierFieldConcepts, ModifierTargetMixin, merge_table_args, omop_index, + optional_concept_fk, + role_fk, ) if TYPE_CHECKING: @@ -36,17 +39,17 @@ class Condition_Occurrence( omop_index(__tablename__, "visit_occurrence_id") ) condition_occurrence_id: so.Mapped[int] = so.mapped_column(primary_key=True) - condition_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + condition_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) condition_start_date: so.Mapped[date] = so.mapped_column(nullable=False) condition_start_datetime: so.Mapped[Optional[datetime]] = so.mapped_column() condition_end_date: so.Mapped[Optional[date]] = so.mapped_column() condition_end_datetime: so.Mapped[Optional[datetime]] = so.mapped_column() - condition_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + condition_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) stop_reason: so.Mapped[Optional[str]] = so.mapped_column(sa.String(20)) condition_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50)) condition_status_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50)) - condition_source_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) - condition_status_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) + condition_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() + condition_status_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() class Condition_OccurrenceContext(ReferenceContext): condition_concept: so.Mapped["Concept"] = ReferenceContext._reference_relationship(target="Concept", local_fk="condition_concept_id", remote_pk="concept_id") # type: ignore[assignment] diff --git a/omop_alchemy/cdm/model/clinical/measurement.py b/omop_alchemy/cdm/model/clinical/measurement.py index aa94954..159d71f 100644 --- a/omop_alchemy/cdm/model/clinical/measurement.py +++ b/omop_alchemy/cdm/model/clinical/measurement.py @@ -5,10 +5,13 @@ from sqlalchemy.ext.hybrid import hybrid_property from typing import Optional from datetime import date, datetime +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( CDMTableBase, cdm_table, + optional_concept_fk, + role_fk, ValueMixin, merge_table_args, omop_index, @@ -27,13 +30,13 @@ class Measurement(Base, CDMTableBase, ValueMixin): measurement_id: so.Mapped[int] = so.mapped_column(primary_key=True) person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) - measurement_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + measurement_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) measurement_date: so.Mapped[date] = so.mapped_column(nullable=False) measurement_datetime: so.Mapped[Optional[datetime]] measurement_time: so.Mapped[Optional[str]] - measurement_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) - operator_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) - unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) + measurement_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) + operator_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() + unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() range_low: so.Mapped[Optional[float]] range_high: so.Mapped[Optional[float]] @@ -43,13 +46,13 @@ class Measurement(Base, CDMTableBase, ValueMixin): visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id")) measurement_source_value: so.Mapped[Optional[str]] - measurement_source_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) + measurement_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() unit_source_value: so.Mapped[Optional[str]] - unit_source_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) + unit_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() value_source_value: so.Mapped[Optional[str]] measurement_event_id: so.Mapped[Optional[int]] - meas_event_field_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), doc="Identifies which OMOP table measurement_event_id refers to",) + meas_event_field_concept_id: so.Mapped[Optional[int]] = optional_concept_fk(doc="Identifies which OMOP table measurement_event_id refers to") @hybrid_property def modifier_of_event_id(self) -> Optional[int]: diff --git a/omop_alchemy/cdm/model/clinical/observation.py b/omop_alchemy/cdm/model/clinical/observation.py index e71bab4..6795f48 100644 --- a/omop_alchemy/cdm/model/clinical/observation.py +++ b/omop_alchemy/cdm/model/clinical/observation.py @@ -5,10 +5,13 @@ from sqlalchemy.ext.hybrid import hybrid_property from typing import Optional from datetime import date, datetime +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( CDMTableBase, cdm_table, + optional_concept_fk, + role_fk, ValueMixin, merge_table_args, omop_index, @@ -26,25 +29,25 @@ class Observation(Base, CDMTableBase, ValueMixin): observation_id: so.Mapped[int] = so.mapped_column(primary_key=True) person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) - observation_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + observation_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) observation_date: so.Mapped[date] = so.mapped_column(nullable=False) observation_datetime: so.Mapped[Optional[datetime]] - observation_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + observation_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) #value_as_number: so.Mapped[Optional[float]] value_as_string: so.Mapped[Optional[str]] - #value_as_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) - qualifier_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) - unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) + #value_as_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() + qualifier_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() + unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id")) visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_occurrence.visit_occurrence_id")) visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id")) observation_source_value: so.Mapped[Optional[str]] - observation_source_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) + observation_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() unit_source_value: so.Mapped[Optional[str]] qualifier_source_value: so.Mapped[Optional[str]] value_source_value: so.Mapped[Optional[str]] observation_event_id: so.Mapped[Optional[int]] - obs_event_field_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id")) + obs_event_field_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() @hybrid_property def modifier_of_event_id(self) -> Optional[int]: diff --git a/omop_alchemy/cdm/model/derived/cohort.py b/omop_alchemy/cdm/model/derived/cohort.py index 987584c..cf1544a 100644 --- a/omop_alchemy/cdm/model/derived/cohort.py +++ b/omop_alchemy/cdm/model/derived/cohort.py @@ -1,15 +1,18 @@ import sqlalchemy as sa import sqlalchemy.orm as so from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, CDMTableBase, + merge_table_args, ) @cdm_table class Cohort(CDMTableBase, Base): __tablename__ = "cohort" + __table_args__ = merge_table_args({"schema": Role.RESULTS.value}) cohort_definition_id: so.Mapped[int] = so.mapped_column(primary_key=True) subject_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/derived/cohort_definition.py b/omop_alchemy/cdm/model/derived/cohort_definition.py index 3cd8198..08cf272 100644 --- a/omop_alchemy/cdm/model/derived/cohort_definition.py +++ b/omop_alchemy/cdm/model/derived/cohort_definition.py @@ -2,12 +2,14 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, CDMTableBase, merge_table_args, omop_index, + role_fk, ) @cdm_table @@ -15,16 +17,17 @@ class Cohort_Definition(CDMTableBase, Base): __tablename__ = "cohort_definition" __table_args__ = merge_table_args( omop_index(__tablename__, "definition_type_concept_id"), - omop_index(__tablename__, "subject_concept_id") + omop_index(__tablename__, "subject_concept_id"), + {"schema": Role.RESULTS.value}, ) cohort_definition_id: so.Mapped[int] = so.mapped_column(primary_key=True) cohort_definition_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) cohort_definition_description: so.Mapped[Optional[str]] = so.mapped_column(sa.Text, nullable=True) - definition_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + definition_type_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) cohort_definition_syntax: so.Mapped[Optional[str]] = so.mapped_column(sa.Text, nullable=True) - subject_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + subject_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) cohort_initiation_date: so.Mapped[Optional[date]] = so.mapped_column(sa.Date, nullable=True) diff --git a/omop_alchemy/cdm/model/derived/condition_era.py b/omop_alchemy/cdm/model/derived/condition_era.py index 3891911..e3dcdcb 100644 --- a/omop_alchemy/cdm/model/derived/condition_era.py +++ b/omop_alchemy/cdm/model/derived/condition_era.py @@ -2,6 +2,7 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, @@ -17,6 +18,7 @@ class Condition_Era(CDMTableBase, Base): __table_args__ = merge_table_args( omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "condition_concept_id"), + {"schema": Role.RESULTS.value}, ) condition_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/derived/dose_era.py b/omop_alchemy/cdm/model/derived/dose_era.py index 85f0975..f1e91b6 100644 --- a/omop_alchemy/cdm/model/derived/dose_era.py +++ b/omop_alchemy/cdm/model/derived/dose_era.py @@ -1,6 +1,7 @@ import sqlalchemy as sa import sqlalchemy.orm as so from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, @@ -17,6 +18,7 @@ class Dose_Era(CDMTableBase, Base): omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), omop_index(__tablename__, "unit_concept_id"), + {"schema": Role.RESULTS.value}, ) dose_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/derived/drug_era.py b/omop_alchemy/cdm/model/derived/drug_era.py index 695c3ca..230525e 100644 --- a/omop_alchemy/cdm/model/derived/drug_era.py +++ b/omop_alchemy/cdm/model/derived/drug_era.py @@ -2,6 +2,7 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, @@ -17,6 +18,7 @@ class Drug_Era(CDMTableBase, Base): __table_args__ = merge_table_args( omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), + {"schema": Role.RESULTS.value}, ) drug_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/derived/observation_period.py b/omop_alchemy/cdm/model/derived/observation_period.py index b0d7bcc..e4914d1 100644 --- a/omop_alchemy/cdm/model/derived/observation_period.py +++ b/omop_alchemy/cdm/model/derived/observation_period.py @@ -1,6 +1,7 @@ import sqlalchemy as sa import sqlalchemy.orm as so from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, @@ -16,6 +17,7 @@ class Observation_Period(CDMTableBase, Base): __table_args__ = merge_table_args( omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "period_type_concept_id"), + {"schema": Role.RESULTS.value}, ) observation_period_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/metadata/cdm_source.py b/omop_alchemy/cdm/model/metadata/cdm_source.py index c02ee60..a561430 100644 --- a/omop_alchemy/cdm/model/metadata/cdm_source.py +++ b/omop_alchemy/cdm/model/metadata/cdm_source.py @@ -2,6 +2,7 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( @@ -9,6 +10,7 @@ CDMTableBase, merge_table_args, omop_index, + role_fk, ) @cdm_table @@ -30,7 +32,7 @@ class CDM_Source(CDMTableBase, Base): cdm_release_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) cdm_version: so.Mapped[Optional[str]] = so.mapped_column(sa.String(10), nullable=True) - cdm_version_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) + cdm_version_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) vocabulary_version: so.Mapped[str] = so.mapped_column(sa.String(20), nullable=False) diff --git a/omop_alchemy/cdm/model/structural/episode_event.py b/omop_alchemy/cdm/model/structural/episode_event.py index 3720c4d..d4719a1 100644 --- a/omop_alchemy/cdm/model/structural/episode_event.py +++ b/omop_alchemy/cdm/model/structural/episode_event.py @@ -4,6 +4,7 @@ from sqlalchemy.orm import Mapper from typing import TYPE_CHECKING, Any, Type from functools import cached_property, cache +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( cdm_table, @@ -15,6 +16,7 @@ ModifierTargetMixin, merge_table_args, omop_index, + role_fk, ) if TYPE_CHECKING: @@ -84,7 +86,7 @@ class Episode_Event(CDMTableBase, Base): episode_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("episode.episode_id"),nullable=False,primary_key=True) event_id: so.Mapped[int] = so.mapped_column(nullable=False,primary_key=True) - episode_event_field_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),nullable=False,primary_key=True) + episode_event_field_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False,primary_key=True) def __repr__(self) -> str: return f"" diff --git a/omop_alchemy/cdm/model/structural/fact_relationship.py b/omop_alchemy/cdm/model/structural/fact_relationship.py index ed40449..a83e1d0 100644 --- a/omop_alchemy/cdm/model/structural/fact_relationship.py +++ b/omop_alchemy/cdm/model/structural/fact_relationship.py @@ -1,12 +1,14 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( CDMTableBase, - cdm_table, + cdm_table, merge_table_args, omop_index, + role_fk, ) @cdm_table @@ -19,19 +21,19 @@ class Fact_Relationship(CDMTableBase, Base): ) domain_concept_id_1: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), primary_key=True, nullable=False, ) fact_id_1: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True, nullable=False) domain_concept_id_2: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), primary_key=True, nullable=False, ) fact_id_2: so.Mapped[int] = so.mapped_column(sa.Integer, primary_key=True, nullable=False) relationship_concept_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("concept.concept_id"), + sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), primary_key=True, nullable=False, ) diff --git a/omop_alchemy/cdm/model/vocabulary/concept.py b/omop_alchemy/cdm/model/vocabulary/concept.py index 5bab7ec..202c3f0 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept.py +++ b/omop_alchemy/cdm/model/vocabulary/concept.py @@ -10,6 +10,7 @@ from .concept_ancestor import Concept_Ancestor from .concept_relationship import Concept_Relationship +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -20,6 +21,7 @@ omop_index, omop_primary_key_index_name, omop_table_options, + role_fk, ) from omop_alchemy.cdm.model.flags import ( StandardConceptFlag, @@ -49,12 +51,13 @@ class Concept( name="ix_concept_concept_name_lower", ), omop_table_options(cluster_on=omop_primary_key_index_name("concept")), + {"schema": Role.VOCAB.value}, ) concept_id: so.Mapped[int] = so.mapped_column(primary_key=True) concept_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) - domain_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("domain.domain_id"), nullable=False) - vocabulary_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("vocabulary.vocabulary_id"), nullable=False) - concept_class_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("concept_class.concept_class_id"), nullable=False) + domain_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "domain.domain_id")), nullable=False) + vocabulary_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "vocabulary.vocabulary_id")), nullable=False) + concept_class_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept_class.concept_class_id")), nullable=False) standard_concept: so.Mapped[Optional[str]] = so.mapped_column(sa.String(1), nullable=True) concept_code: so.Mapped[str] = so.mapped_column(sa.String(50), nullable=False) valid_start_date: so.Mapped[date] = so.mapped_column(sa.Date(), nullable=False) @@ -160,4 +163,8 @@ class ConceptView(Concept, ConceptContext): Avoid in tight loops or ETL paths. """ __tablename__ = "concept" + # Must match Concept.__table_args__'s schema exactly: same (schema, name) + # key is what makes SQLAlchemy reuse Concept's own Table object here + # instead of building a second, distinct one with no FK link between them. + __table_args__ = {"schema": Role.VOCAB.value} __mapper_args__ = {"concrete": False} diff --git a/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py b/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py index d933377..2d7be7c 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -7,6 +8,7 @@ CDMTableBase, merge_table_args, omop_index, + role_fk, ) @cdm_table @@ -15,9 +17,10 @@ class Concept_Ancestor(Base, ReferenceTable, CDMTableBase): __table_args__ = merge_table_args( omop_index(__tablename__, "ancestor_concept_id", cluster=True), omop_index(__tablename__, "descendant_concept_id"), + {"schema": Role.VOCAB.value}, ) - ancestor_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) - descendant_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) + ancestor_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) + descendant_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) min_levels_of_separation: so.Mapped[int] = so.mapped_column(nullable=False) max_levels_of_separation: so.Mapped[int] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/concept_class.py b/omop_alchemy/cdm/model/vocabulary/concept_class.py index c76faf5..05b6f74 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_class.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_class.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -8,6 +9,7 @@ merge_table_args, omop_primary_key_index_name, omop_table_options, + role_fk, ) @cdm_table @@ -15,10 +17,11 @@ class Concept_Class(Base, ReferenceTable, CDMTableBase): __tablename__ = "concept_class" __table_args__ = merge_table_args( omop_table_options(cluster_on=omop_primary_key_index_name("concept_class")), + {"schema": Role.VOCAB.value}, ) concept_class_id: so.Mapped[str] = so.mapped_column(sa.String(20), primary_key=True) concept_class_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) - concept_class_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),nullable=False,) + concept_class_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False,) def __repr__(self): return f"" diff --git a/omop_alchemy/cdm/model/vocabulary/concept_relationship.py b/omop_alchemy/cdm/model/vocabulary/concept_relationship.py index 7b08f32..ca76eb0 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_relationship.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_relationship.py @@ -1,6 +1,7 @@ import sqlalchemy as sa import sqlalchemy.orm as so from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -8,6 +9,7 @@ CDMTableBase, merge_table_args, omop_index, + role_fk, ) from omop_alchemy.cdm.model.flags import InvalidReasonMixin @@ -23,9 +25,10 @@ class Concept_Relationship( omop_index(__tablename__, "concept_id_1", cluster=True), omop_index(__tablename__, "concept_id_2"), omop_index(__tablename__, "relationship_id"), + {"schema": Role.VOCAB.value}, ) - concept_id_1: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) - concept_id_2: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) - relationship_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("relationship.relationship_id"),primary_key=True) + concept_id_1: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) + concept_id_2: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) + relationship_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "relationship.relationship_id")),primary_key=True) valid_start_date: so.Mapped[date] = so.mapped_column(nullable=False) valid_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/concept_synonym.py b/omop_alchemy/cdm/model/vocabulary/concept_synonym.py index fd124fa..762c5ac 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_synonym.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_synonym.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -7,6 +8,7 @@ CDMTableBase, merge_table_args, omop_index, + role_fk, ) @cdm_table @@ -21,7 +23,8 @@ class Concept_Synonym(Base, ReferenceTable, CDMTableBase): sa.func.lower(sa.column("concept_synonym_name")), name="ix_concept_synonym_concept_synonym_name_lower", ), + {"schema": Role.VOCAB.value}, ) - concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) + concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) concept_synonym_name: so.Mapped[str] = so.mapped_column(sa.String(1000),primary_key=True) - language_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) + language_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) diff --git a/omop_alchemy/cdm/model/vocabulary/domain.py b/omop_alchemy/cdm/model/vocabulary/domain.py index db1eb31..3c7de50 100644 --- a/omop_alchemy/cdm/model/vocabulary/domain.py +++ b/omop_alchemy/cdm/model/vocabulary/domain.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -8,6 +9,7 @@ merge_table_args, omop_primary_key_index_name, omop_table_options, + role_fk, ) @cdm_table @@ -15,10 +17,11 @@ class Domain(Base, ReferenceTable, CDMTableBase): __tablename__ = "domain" __table_args__ = merge_table_args( omop_table_options(cluster_on=omop_primary_key_index_name("domain")), + {"schema": Role.VOCAB.value}, ) domain_id: so.Mapped[str] = so.mapped_column(sa.String(20), primary_key=True) domain_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) - domain_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),nullable=False) + domain_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False) def __repr__(self): return f'' diff --git a/omop_alchemy/cdm/model/vocabulary/drug_strength.py b/omop_alchemy/cdm/model/vocabulary/drug_strength.py index 668224d..f3ccdd3 100644 --- a/omop_alchemy/cdm/model/vocabulary/drug_strength.py +++ b/omop_alchemy/cdm/model/vocabulary/drug_strength.py @@ -2,6 +2,7 @@ import sqlalchemy.orm as so from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -9,6 +10,8 @@ CDMTableBase, merge_table_args, omop_index, + optional_concept_fk, + role_fk, ) from omop_alchemy.cdm.model.flags import InvalidReasonMixin @@ -29,16 +32,17 @@ class Drug_Strength( __table_args__ = merge_table_args( omop_index(__tablename__, "drug_concept_id", cluster=True), omop_index(__tablename__, "ingredient_concept_id"), + {"schema": Role.VOCAB.value}, ) - drug_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) - ingredient_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),primary_key=True) + drug_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) + ingredient_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) amount_value: so.Mapped[Optional[float]] = so.mapped_column(sa.Float, nullable=True) - amount_unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + amount_unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() numerator_value: so.Mapped[Optional[float]] = so.mapped_column(sa.Float, nullable=True) - numerator_unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + numerator_unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() denominator_value: so.Mapped[Optional[float]] = so.mapped_column(sa.Float, nullable=True) - denominator_unit_concept_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=True) + denominator_unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() box_size: so.Mapped[Optional[int]] = so.mapped_column(sa.Integer, nullable=True) valid_start_date: so.Mapped[date] = so.mapped_column(nullable=False) valid_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/relationship.py b/omop_alchemy/cdm/model/vocabulary/relationship.py index fd72e77..ba2c277 100644 --- a/omop_alchemy/cdm/model/vocabulary/relationship.py +++ b/omop_alchemy/cdm/model/vocabulary/relationship.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -8,6 +9,7 @@ merge_table_args, omop_primary_key_index_name, omop_table_options, + role_fk, ) from omop_alchemy.cdm.model.flags import BooleanFlag, normalised_flag_expr, normalised_flag @@ -16,13 +18,14 @@ class Relationship(Base, ReferenceTable, CDMTableBase): __tablename__ = "relationship" __table_args__ = merge_table_args( omop_table_options(cluster_on=omop_primary_key_index_name("relationship")), + {"schema": Role.VOCAB.value}, ) relationship_id: so.Mapped[str] = so.mapped_column(sa.String(20), primary_key=True) relationship_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) is_hierarchical: so.Mapped[str] = so.mapped_column(sa.String(1), nullable=False) defines_ancestry: so.Mapped[str] = so.mapped_column(sa.String(1), nullable=False) - reverse_relationship_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("relationship.relationship_id"),nullable=False) - relationship_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),nullable=False,) + reverse_relationship_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "relationship.relationship_id")),nullable=False) + relationship_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False,) def __repr__(self): return f"" diff --git a/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py b/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py index 3b3da12..9516348 100644 --- a/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py +++ b/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py @@ -3,6 +3,7 @@ from typing import Optional from datetime import date +from oa_configurator import Role from orm_loader.helpers import Base from orm_loader.registry import ValidationIssue from omop_alchemy.cdm.base import ( @@ -12,6 +13,7 @@ DatedEvent, merge_table_args, omop_index, + role_fk, ) from omop_alchemy.cdm.model.flags import InvalidReasonMixin @@ -36,14 +38,15 @@ class Source_To_Concept_Map( omop_index(__tablename__, "source_vocabulary_id"), omop_index(__tablename__, "target_vocabulary_id"), omop_index(__tablename__, "source_code"), + {"schema": Role.VOCAB.value}, ) source_code: so.Mapped[str] = so.mapped_column(sa.String(50),primary_key=True) source_concept_id: so.Mapped[int] = so.mapped_column(sa.Integer,primary_key=True,doc="0 or >= 2,000,000,000 for site-specific concepts") source_vocabulary_id: so.Mapped[str] = so.mapped_column(sa.String(20),primary_key=True) source_code_description: so.Mapped[Optional[str]] = so.mapped_column(sa.String(255), nullable=True) - target_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"), nullable=False) - target_vocabulary_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey("vocabulary.vocabulary_id"),nullable=False) + target_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) + target_vocabulary_id: so.Mapped[str] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "vocabulary.vocabulary_id")),nullable=False) valid_start_date: so.Mapped[date] = so.mapped_column(nullable=False) valid_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/vocabulary.py b/omop_alchemy/cdm/model/vocabulary/vocabulary.py index 4efa788..d670872 100644 --- a/omop_alchemy/cdm/model/vocabulary/vocabulary.py +++ b/omop_alchemy/cdm/model/vocabulary/vocabulary.py @@ -13,6 +13,7 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( ReferenceTable, @@ -21,6 +22,7 @@ merge_table_args, omop_primary_key_index_name, omop_table_options, + role_fk, ) @cdm_table @@ -28,12 +30,13 @@ class Vocabulary(Base, ReferenceTable, CDMTableBase): __tablename__ = "vocabulary" __table_args__ = merge_table_args( omop_table_options(cluster_on=omop_primary_key_index_name("vocabulary")), + {"schema": Role.VOCAB.value}, ) vocabulary_id: so.Mapped[str] = so.mapped_column(sa.String(20), primary_key=True) vocabulary_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) vocabulary_reference: so.Mapped[Optional[str]] = so.mapped_column(sa.String(255), nullable=True) vocabulary_version: so.Mapped[Optional[str]] = so.mapped_column(sa.String(255), nullable=True) - vocabulary_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("concept.concept_id"),nullable=False,) + vocabulary_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False,) def __repr__(self): return f"" diff --git a/omop_alchemy/config.py b/omop_alchemy/config.py index 09bb7d9..a4b33a9 100644 --- a/omop_alchemy/config.py +++ b/omop_alchemy/config.py @@ -26,9 +26,18 @@ class OmopAlchemyConfig(PackageConfigBase): ---------- cdm_db : str Name of the ``[databases.*]`` entry holding the CDM database. - test_cdm_db : str, optional + test_cdm_db_pg : str, optional Name of the ``[databases.*]`` entry holding the test CDM database, - marked ``RefTo(CDMDatabaseConfig, is_test=True)``. + marked ``RefTo(CDMDatabaseConfig, is_test=True)``. Must resolve to a + real PostgreSQL connection; used for real integration testing of + Postgres-only behavior (FK triggers, catalog queries, ALTER, etc.). + test_cdm_db_sqlite : str, optional + Same shape as ``test_cdm_db_pg``, for tests that must always run + against SQLite specifically (dialect-behavior tests), regardless of + what ``test_cdm_db_pg`` happens to be configured to. Left + unconfigured by design in every environment, since + ``isolated_test_database(..., dialect="sqlite")`` provisions a + disposable instance with no config needed at all. Notes ----- @@ -40,9 +49,21 @@ class OmopAlchemyConfig(PackageConfigBase): extra_logging_namespaces: ClassVar[tuple[str, ...]] = ("orm_loader",) cdm_db: Annotated[str, RefTo(CDMDatabaseConfig)] = "cdm_db" - test_cdm_db: Annotated[ + test_cdm_db_pg: Annotated[ str | None, RefTo(CDMDatabaseConfig, is_test=True) - ] = None + ] = Field( + default=None, + description="Real PostgreSQL test CDM database, for Postgres-only integration testing.", + ) + test_cdm_db_sqlite: Annotated[ + str | None, RefTo(CDMDatabaseConfig, is_test=True) + ] = Field( + default=None, + description=( + "Disposable SQLite test database; left unconfigured by design " + "(isolated_test_database(..., dialect='sqlite') provisions one automatically)." + ), + ) athena_source_path: str | None = Field( default=None, diff --git a/omop_alchemy/maintenance/_cli_utils.py b/omop_alchemy/maintenance/_cli_utils.py index a907c9a..e7e2dba 100644 --- a/omop_alchemy/maintenance/_cli_utils.py +++ b/omop_alchemy/maintenance/_cli_utils.py @@ -10,32 +10,20 @@ import sqlalchemy as sa import typer -from orm_loader.backends import STAGING_SCHEMA +from oa_configurator import register_reserved_schema from sqlalchemy.exc import SQLAlchemyError from .tables import TableCategory from .ui import console, render_error, render_command_header -from ..backends import BackendNotSupportedError, resolve_backend +from ..backends import BackendNotSupportedError _F = TypeVar("_F", bound=Callable[..., Any]) -class ReservedSchema(StrEnum): - """Schema names reserved for OMOP_Alchemy/orm-loader internal bookkeeping. - A user-configured db_schema may never collide with one of these. - """ - - STAGING = STAGING_SCHEMA - MAINTENANCE = "omop_alchemy_maintenance" - +MAINTENANCE_SCHEMA: str = "omop_alchemy_maintenance" -def reject_reserved_schema(db_schema: str | None) -> None: - """Raise if db_schema collides with a schema name reserved for internal bookkeeping.""" - if db_schema in set(ReservedSchema): - raise RuntimeError( - f"db_schema cannot be {db_schema!r}: reserved for OMOP_Alchemy/orm-loader internal use." - ) +register_reserved_schema(MAINTENANCE_SCHEMA, owner="omop_alchemy") class Severity(StrEnum): @@ -122,11 +110,18 @@ def __init__(self, code: str, severity: Severity): @dataclass(frozen=True) class _ConnContext: - """Connection context derived from the oa_configurator resolved resource.""" + """Connection context derived from the oa_configurator resolved resource. + + ``vocab_engine`` is the same object as ``engine`` unless ``vocab_connection`` + names a physically different server; only DDL creating vocab-role tables + needs it instead of the primary engine. + """ db_schema: str | None engine_url: str = "" resource_name: str = "" athena_source: str | None = None # from OmopAlchemyConfig.athena_source_path + vocab_engine: sa.Engine | None = None + vocab_schema: str | None = None # ── Decorator ───────────────────────────────────────────────────────────────── @@ -155,12 +150,14 @@ def wrapper(**kwargs: Any) -> Any: try: from ..config import create_cdm_engine, get_cdm_context pkg_config, resolved = get_cdm_context() - reject_reserved_schema(resolved.schema_name) engine = create_cdm_engine(resolved) + vocab_engine = resolved.vocab_engine_for(engine) conn = _ConnContext( db_schema=resolved.schema_name, engine_url=engine.url.render_as_string(hide_password=True), resource_name=pkg_config.cdm_db, + vocab_engine=vocab_engine, + vocab_schema=resolved.vocab_schema, athena_source=pkg_config.athena_source_path, ) console.print( @@ -211,21 +208,6 @@ def wrapper(**kwargs: Any) -> Any: # ── Helpers ─────────────────────────────────────────────────────────────────── -def ensure_schema(engine: sa.Engine, schema: str | None) -> None: - """Create the schema in the database if it does not already exist. - - Delegates to the active backend so behaviour is correct for each dialect. - No-op when schema is None or ``"public"``, or on backends that don't - support named schemas (e.g. SQLite). - """ - if not schema or schema == "public": - return - backend = resolve_backend(engine) - with engine.connect() as conn: - backend.ensure_schema(conn, schema) - conn.commit() - - def handle_error(exc: Exception) -> None: if isinstance(exc, BackendNotSupportedError): console.print(render_error(f"Not supported: {exc}")) diff --git a/omop_alchemy/maintenance/cli_backup.py b/omop_alchemy/maintenance/cli_backup.py index 765be4c..d9acc22 100644 --- a/omop_alchemy/maintenance/cli_backup.py +++ b/omop_alchemy/maintenance/cli_backup.py @@ -12,7 +12,7 @@ import typer from ..backends import resolve_backend, require_backend_support, backend_support_note -from ._cli_utils import Status, dry_label, dry_status, omop_command, reject_reserved_schema +from ._cli_utils import Status, dry_label, dry_status, omop_command from .ui import ( console, render_backup_result, @@ -65,14 +65,13 @@ def create_database_backup( dry_run: bool = False, ) -> BackupResult: """Create a database backup artifact at output_path. Runs the subprocess unless dry_run is True.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "prepare_backup", "Database backup") resolved_output_path = Path(output_path) if output_path is not None else _default_output_path(backup_format) resolved_output_path = resolved_output_path.expanduser().resolve() tool_path, command, env, database_name = backend.prepare_backup( - engine, str(resolved_output_path), backup_format.value, db_schema + engine, str(resolved_output_path), backup_format.value ) if not dry_run: @@ -112,14 +111,13 @@ def restore_database_backup( dry_run: bool = False, ) -> BackupResult: """Restore a database backup. Runs the subprocess unless dry_run is True.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "prepare_restore", "Database restore") resolved_input_path = Path(input_path).expanduser().resolve() if not resolved_input_path.exists(): raise RuntimeError(f"Backup artifact not found: {resolved_input_path}") tool_path, command, env, database_name = backend.prepare_restore( - engine, str(resolved_input_path), backup_format.value, db_schema + engine, str(resolved_input_path), backup_format.value ) if not dry_run: diff --git a/omop_alchemy/maintenance/cli_foreign_keys.py b/omop_alchemy/maintenance/cli_foreign_keys.py index a17d19f..1d69cd3 100644 --- a/omop_alchemy/maintenance/cli_foreign_keys.py +++ b/omop_alchemy/maintenance/cli_foreign_keys.py @@ -8,7 +8,7 @@ import typer from ..backends import Backend, resolve_backend, require_backend_support, backend_support_note -from ._cli_utils import Status, dry_label, dry_status, omop_command, reject_reserved_schema +from ._cli_utils import Status, dry_label, dry_status, omop_command from .tables import ( TableCategory, existing_maintenance_tables, @@ -179,7 +179,6 @@ def _collect_strict_validation_failures( str(referred_table), list(constrained_columns), list(referred_columns), - db_schema, ) if violation_count == 0: @@ -286,7 +285,6 @@ def manage_foreign_key_triggers( strict: bool = False, ) -> list[ForeignKeyManagementResult]: """Enable or disable RI trigger enforcement. With strict=True, aborts on any FK violation.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "toggle_fk_triggers", "FK trigger management") @@ -333,7 +331,7 @@ def manage_foreign_key_triggers( } for target in targets: if not dry_run: - backend.toggle_fk_triggers(connection, target.table_name, db_schema, enable=enable) + backend.toggle_fk_triggers(connection, target.table_name, enable=enable) results.append( ForeignKeyManagementResult( @@ -370,7 +368,7 @@ def collect_foreign_key_trigger_status( with engine.connect() as connection: for target in targets: disabled_count, enabled_count = backend.get_fk_trigger_counts( - connection, target.table_name, db_schema + connection, target.table_name ) results.append( ForeignKeyStatusResult( diff --git a/omop_alchemy/maintenance/cli_fulltext.py b/omop_alchemy/maintenance/cli_fulltext.py index 13b788c..c4793a4 100644 --- a/omop_alchemy/maintenance/cli_fulltext.py +++ b/omop_alchemy/maintenance/cli_fulltext.py @@ -11,7 +11,7 @@ from ..backends import backend_support_note as _backend_support_note from ..backends import resolve_backend, require_backend_support from ..backends.base import FullTextError -from ._cli_utils import Status, dry_label, dry_status, omop_command, reject_reserved_schema +from ._cli_utils import Status, dry_label, dry_status, omop_command from .ui import ( console, render_fulltext_results, @@ -57,7 +57,6 @@ def install_fulltext_columns( dry_run: bool = False, ) -> tuple[FullTextResult, ...]: """Install tsvector sidecar columns (and optionally GIN indexes) on OMOP vocabulary tables.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "install_fulltext_on_table", "Full-text search") targets = backend.fulltext_targets @@ -71,7 +70,6 @@ def install_fulltext_columns( table_name=cfg.table_name, vector_column_name=cfg.vector_column_name, index_name=cfg.index_name, - db_schema=db_schema, create_indexes=create_indexes, fastupdate=fastupdate, ) @@ -110,7 +108,6 @@ def populate_fulltext_columns( dry_run: bool = False, ) -> tuple[FullTextResult, ...]: """Populate tsvector sidecar columns with pre-computed search vectors.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "populate_fulltext_on_table", "Full-text search") targets = backend.fulltext_targets @@ -125,7 +122,6 @@ def populate_fulltext_columns( table_name=cfg.table_name, vector_column_name=cfg.vector_column_name, source_column_name=cfg.source_column_name, - db_schema=db_schema, regconfig=regconfig, ) backend.register_fulltext_metadata() @@ -160,7 +156,6 @@ def drop_fulltext_columns( dry_run: bool = False, ) -> tuple[FullTextResult, ...]: """Remove tsvector sidecar columns and their associated GIN indexes.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "drop_fulltext_on_table", "Full-text search") targets = backend.fulltext_targets @@ -174,7 +169,6 @@ def drop_fulltext_columns( table_name=cfg.table_name, vector_column_name=cfg.vector_column_name, index_name=cfg.index_name, - db_schema=db_schema, drop_indexes=drop_indexes, ) backend.unregister_fulltext_metadata() diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index b639b5f..841dba0 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -10,14 +10,15 @@ from sqlalchemy.exc import DBAPIError, IntegrityError import typer +from oa_configurator import ensure_schema, supports_schemas + from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY -from ..backends import Backend, resolve_backend, backend_supports -from ._cli_utils import ReservedSchema, Status, dry_label, dry_status, omop_command, reject_reserved_schema +from ..backends import resolve_backend, backend_supports +from ._cli_utils import MAINTENANCE_SCHEMA, Status, dry_label, dry_status, omop_command from .tables import ( MaintenanceTable, TableCategory, - schema_adjusted_metadata, select_omop_tables, ) from .ui import ( @@ -219,23 +220,22 @@ def _schema_key(db_schema: str | None) -> str: return db_schema or "" -def get_bookkeeping_schema(backend: Backend) -> str | None: +def get_bookkeeping_schema(connection: sa.Connection) -> str | None: """Return the reserved schema name for the dropped-index bookkeeping table. Parameters ---------- - backend : Backend - The resolved database backend. + connection : sqlalchemy.Connection + The connection the bookkeeping table would be created on. Returns ------- str or None - ReservedSchema.MAINTENANCE.value on backends that override - Backend.ensure_schema() (i.e. support named schemas, like - PostgreSQL), or None on backends that don't (like SQLite). + MAINTENANCE_SCHEMA on a dialect with a genuine multi-schema concept + (like PostgreSQL), or None on one that doesn't (like SQLite). """ - if backend_supports(backend, "ensure_schema"): - return ReservedSchema.MAINTENANCE.value + if supports_schemas(connection): + return MAINTENANCE_SCHEMA return None @@ -284,7 +284,6 @@ def _dropped_indexes_table(bookkeeping_schema: str | None) -> sa.Table: def _record_captured_index( connection: sa.Connection, - backend: Backend, *, table_name: str, db_schema: str | None, @@ -307,8 +306,6 @@ def _record_captured_index( ---------- connection : sqlalchemy.Connection Open connection/transaction the capture is recorded on. - backend : Backend - The resolved database backend. table_name : str Name of the table the foreign index belongs to. db_schema : str or None @@ -327,8 +324,8 @@ def _record_captured_index( True if the capture was recorded, False if a pending capture already existed for this table/schema/column-set/uniqueness. """ - bookkeeping_schema = get_bookkeeping_schema(backend) - backend.ensure_schema(connection, bookkeeping_schema) + bookkeeping_schema = get_bookkeeping_schema(connection) + ensure_schema(connection, bookkeeping_schema) bookkeeping_table = _dropped_indexes_table(bookkeeping_schema) bookkeeping_table.create(bind=connection, checkfirst=True) @@ -360,7 +357,6 @@ def _record_captured_index( def _peek_captured_index( connection: sa.Connection, - backend: Backend, *, table_name: str, db_schema: str | None, @@ -377,8 +373,6 @@ def _peek_captured_index( ---------- connection : sqlalchemy.Connection Open connection the lookup is performed on. - backend : Backend - The resolved database backend. table_name : str Name of the table the index belongs to. db_schema : str or None @@ -400,7 +394,7 @@ def _peek_captured_index( The matched bookkeeping row, for use in a later delete by id. None if nothing is captured for this table/schema/column-set/uniqueness. """ - bookkeeping_schema = get_bookkeeping_schema(backend) + bookkeeping_schema = get_bookkeeping_schema(connection) inspector = sa.inspect(connection) if not inspector.has_table(_DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema): return None, None, None @@ -420,7 +414,6 @@ def _peek_captured_index( def _restore_captured_index( connection: sa.Connection, - backend: Backend, *, table_name: str, db_schema: str | None, @@ -435,8 +428,6 @@ def _restore_captured_index( ---------- connection : sqlalchemy.Connection Open connection/transaction the index is created on. - backend : Backend - The resolved database backend. table_name : str Name of the table to recreate the index on. db_schema : str or None @@ -465,7 +456,6 @@ def _restore_captured_index( """ restored_index_name, bookkeeping_table, row = _peek_captured_index( connection=connection, - backend=backend, table_name=table_name, db_schema=db_schema, column_names=column_names, @@ -473,10 +463,10 @@ def _restore_captured_index( ) if restored_index_name is None or bookkeeping_table is None or row is None: return None - + # A lightweight, untyped Table (no autoload_with reflection) is sufficient: # CREATE INDEX DDL only needs column names, not real types, PKs, FKs, or - # constraints -- reflecting the whole table would cost several extra + # constraints. Reflecting the whole table would cost several extra # catalog round-trips to fetch metadata this function never uses. lightweight_table = sa.Table( table_name, sa.MetaData(), @@ -501,22 +491,16 @@ def _restore_captured_index( def _schema_metadata_indexes( tables: list[MaintenanceTable], - db_schema: str | None, ) -> dict[tuple[str, str], sa.Index]: - """Return a (table_name, index_name) → Index mapping from ORM metadata, adjusted for db_schema if provided.""" - indexes: dict[tuple[str, str], sa.Index] = {} - - if db_schema is None: - for table in tables: - for index in table.table.indexes: - indexes[(table.table_name, str(index.name))] = index - return indexes - - _, copied_tables = schema_adjusted_metadata(tables, db_schema=db_schema) - for table_name, table in copied_tables.items(): - for index in table.indexes: - indexes[(table_name, str(index.name))] = index + """Return a (table_name, index_name) -> Index mapping from ORM metadata. + Index name/columns don't depend on which schema a table is tagged with, + so this reads straight off each table's own ORM metadata. + """ + indexes: dict[tuple[str, str], sa.Index] = {} + for table in tables: + for index in table.table.indexes: + indexes[(table.table_name, str(index.name))] = index return indexes @@ -674,11 +658,10 @@ def manage_indexes( cluster: bool = True, ) -> list[IndexManagementResult]: """Create or drop all ORM-defined indexes. CLUSTERs tables when enabling and cluster=True.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) inspector = sa.inspect(engine) selected_tables = select_omop_tables(vocabulary_included=vocabulary_included) - metadata_indexes = _schema_metadata_indexes(selected_tables, db_schema) + metadata_indexes = _schema_metadata_indexes(selected_tables) clustering_supported = backend_supports(backend, "cluster_table") results: list[IndexManagementResult] = [] @@ -731,7 +714,7 @@ def manage_indexes( with connection_factory() as connection: if not enable: if not dry_run: - existed_before_drop = backend.index_exists(connection, index_name, db_schema) + existed_before_drop = backend.index_exists(connection, index_name) else: existed_before_drop = exists if not existed_before_drop: @@ -740,21 +723,21 @@ def manage_indexes( if equivalent_name is not None: if not dry_run: captured = _record_captured_index( - connection, backend, + connection, table_name=table.table_name, db_schema=db_schema, index_name=equivalent_name, column_names=column_names, unique=unique, ) else: pending_capture, _, _ = _peek_captured_index( - connection, backend, + connection, table_name=table.table_name, db_schema=db_schema, column_names=column_names, unique=unique, ) captured = pending_capture is None if captured: if not dry_run: - backend.drop_index_if_exists(connection, equivalent_name, db_schema) + backend.drop_index_if_exists(connection, equivalent_name) outcome = _IndexOutcome( status=dry_status(dry_run, Status.CAPTURED), detail=dry_label( @@ -806,19 +789,19 @@ def manage_indexes( physical_name=index_name, ) elif not dry_run: - backend.drop_index_if_exists(connection, index_name, db_schema) + backend.drop_index_if_exists(connection, index_name) # outcome stays default: applied / "metadata-defined index dropped" # dry-run, existed_before_drop True: outcome stays default ("would be dropped") else: if not dry_run: restored_name = _restore_captured_index( - connection, backend, + connection, table_name=table.table_name, db_schema=db_schema, column_names=column_names, unique=unique, ) else: restored_name, _, _ = _peek_captured_index( - connection, backend, + connection, table_name=table.table_name, db_schema=db_schema, column_names=column_names, unique=unique, ) @@ -925,7 +908,7 @@ def manage_indexes( else: if not dry_run: with engine.begin() as connection: - backend.cluster_table(connection, table.table_name, physical_cluster_name, db_schema) + backend.cluster_table(connection, table.table_name, physical_cluster_name) clustered_now = True results.append( @@ -945,7 +928,7 @@ def manage_indexes( if not dry_run and (created_any or clustered_now): with engine.connect() as connection: - backend.analyze_table(connection, table.table_name, db_schema) + backend.analyze_table(connection, table.table_name) connection.commit() return results @@ -1068,9 +1051,9 @@ def cluster_tables_command( if not dry_run: with engine.begin() as connection: - backend.cluster_table(connection, table.table_name, physical_cluster_name, conn.db_schema) + backend.cluster_table(connection, table.table_name, physical_cluster_name) with engine.connect() as connection: - backend.analyze_table(connection, table.table_name, conn.db_schema) + backend.analyze_table(connection, table.table_name) connection.commit() results.append( diff --git a/omop_alchemy/maintenance/cli_schema.py b/omop_alchemy/maintenance/cli_schema.py index b82e2dc..9ab476a 100644 --- a/omop_alchemy/maintenance/cli_schema.py +++ b/omop_alchemy/maintenance/cli_schema.py @@ -150,6 +150,7 @@ def create_missing_tables_command( with console.status("Creating missing tables..."): results = create_missing_tables( engine, + vocab_engine=conn.vocab_engine, db_schema=conn.db_schema, vocabulary_included=vocabulary_included, dry_run=dry_run, diff --git a/omop_alchemy/maintenance/cli_schema_reconcile.py b/omop_alchemy/maintenance/cli_schema_reconcile.py index 1fa5857..57d2f01 100644 --- a/omop_alchemy/maintenance/cli_schema_reconcile.py +++ b/omop_alchemy/maintenance/cli_schema_reconcile.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import sqlalchemy as sa +from oa_configurator import supports_schemas from sqlalchemy.engine.interfaces import ReflectedForeignKeyConstraint, ReflectedIndex from ..backends import backend_supports, resolve_backend @@ -147,6 +148,7 @@ def reconcile_schema( () if vocabulary_included else (TableCategory.VOCABULARY,) ) _backend = resolve_backend(engine) + _cross_schema_fk_supported = supports_schemas(engine) selected_tables = select_maintenance_tables(exclude_categories=excluded_categories) inspector = sa.inspect(engine) all_issues: list[ReconciliationIssue] = [] @@ -276,6 +278,12 @@ def reconcile_schema( for signature, constraint in expected_fks.items(): if signature not in actual_fks: + if ( + not _cross_schema_fk_supported + and constraint.referred_table.schema != expected_table.schema + ): + # SQLite can never create an inline FK crossing a schema boundary + continue constrained_columns, referred_table, referred_columns = signature table_issues.append( ReconciliationIssue( @@ -384,7 +392,6 @@ def reconcile_schema( actual_cluster = _backend.get_clustered_index_name( connection, maintenance_table.table_name, - db_schema, ) if expected_cluster != actual_cluster: # May be a rename, not drift, so treat like a renamed index. diff --git a/omop_alchemy/maintenance/cli_schema_summary.py b/omop_alchemy/maintenance/cli_schema_summary.py index 14e7f45..58c71ea 100644 --- a/omop_alchemy/maintenance/cli_schema_summary.py +++ b/omop_alchemy/maintenance/cli_schema_summary.py @@ -6,7 +6,8 @@ import sqlalchemy as sa -from .tables import TableCategory, qualified_table_name, select_omop_tables +from oa_configurator import qualified +from .tables import TableCategory, select_omop_tables @dataclass(frozen=True) @@ -44,7 +45,7 @@ def collect_data_summary( row_count = int( connection.execute( sa.text( - f"SELECT COUNT(*) FROM {qualified_table_name(table.table_name, db_schema)}" + f"SELECT COUNT(*) FROM {qualified(connection, table.table_name)}" ) ).scalar_one() ) diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index 83f2473..a6a5227 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -6,13 +6,13 @@ import sqlalchemy as sa -from ._cli_utils import Status, dry_label, dry_status, ensure_schema, reject_reserved_schema +from oa_configurator import Role, ensure_schema +from orm_loader.helpers import Base +from ._cli_utils import Status, dry_label, dry_status from .tables import ( MaintenanceTable, TableCategory, - collect_maintenance_tables, missing_maintenance_tables, - schema_adjusted_metadata, ) @@ -57,12 +57,21 @@ def collect_missing_tables( def create_missing_tables( engine: sa.Engine, *, + vocab_engine: sa.Engine | None = None, db_schema: str | None = None, vocabulary_included: bool = True, dry_run: bool = False, ) -> list[TableCreationResult]: - """Create any ORM-managed tables missing from the target database. Skips tables with unresolved FK dependencies.""" - reject_reserved_schema(db_schema) + """Create any ORM-managed tables missing from the target database. Skips tables with unresolved FK dependencies. + + Parameters + ---------- + vocab_engine : sqlalchemy.Engine, optional + Engine for vocab-role tables, when ``vocab_connection`` names a + physically different server than ``engine``. Defaults to ``engine`` + (the common, same-connection case). + """ + vocab_engine = vocab_engine if vocab_engine is not None else engine if not dry_run: ensure_schema(engine, db_schema) inspector = sa.inspect(engine) @@ -92,36 +101,52 @@ def create_missing_tables( ] results: list[TableCreationResult] = [] - with engine.begin() as connection: - if creatable_tables and not dry_run: - metadata, adjusted_tables = schema_adjusted_metadata( - collect_maintenance_tables(), - db_schema=db_schema, - ) - metadata.create_all( - bind=connection, - tables=[adjusted_tables[table.table_name] for table in creatable_tables], - checkfirst=True, - ) - - for maintenance_table in missing_tables: - blocked = blocked_dependencies.get(maintenance_table.table_name) - results.append( - TableCreationResult( - table_name=maintenance_table.table_name, - category=maintenance_table.category, - model_name=maintenance_table.model_name, - status=( - Status.BLOCKED - if blocked is not None - else dry_status(dry_run, applied=Status.CREATED) - ), - detail=( - "table blocked by unresolved dependencies: " + ", ".join(blocked) - if blocked is not None - else dry_label(dry_run, "table would be created from ORM metadata", "table created from ORM metadata") - ), + if creatable_tables and not dry_run: + all_tables = [table.table for table in creatable_tables] + if vocab_engine is engine: + # One call: create_all's dependency sort and FK-deferral must see every table together. + with engine.begin() as connection: + Base.metadata.create_all( + bind=connection, tables=all_tables, checkfirst=True ) + else: + # Split physical connections: a cross-boundary FK can't be created here at all; + # that failure surfaces from create_all itself rather than being masked. + vocab_tables = [ + table for table in all_tables if table.schema == Role.VOCAB.value + ] + other_tables = [ + table for table in all_tables if table.schema != Role.VOCAB.value + ] + if other_tables: + with engine.begin() as connection: + Base.metadata.create_all( + bind=connection, tables=other_tables, checkfirst=True + ) + if vocab_tables: + with vocab_engine.begin() as vocab_connection: + Base.metadata.create_all( + bind=vocab_connection, tables=vocab_tables, checkfirst=True + ) + + for maintenance_table in missing_tables: + blocked = blocked_dependencies.get(maintenance_table.table_name) + results.append( + TableCreationResult( + table_name=maintenance_table.table_name, + category=maintenance_table.category, + model_name=maintenance_table.model_name, + status=( + Status.BLOCKED + if blocked is not None + else dry_status(dry_run, applied=Status.CREATED) + ), + detail=( + "table blocked by unresolved dependencies: " + ", ".join(blocked) + if blocked is not None + else dry_label(dry_run, "table would be created from ORM metadata", "table created from ORM metadata") + ), ) + ) return results diff --git a/omop_alchemy/maintenance/cli_tables.py b/omop_alchemy/maintenance/cli_tables.py index 7c7fa5e..0cac324 100644 --- a/omop_alchemy/maintenance/cli_tables.py +++ b/omop_alchemy/maintenance/cli_tables.py @@ -7,11 +7,11 @@ import sqlalchemy as sa import typer +from oa_configurator import autocommit_connection, qualified from ..backends import resolve_backend, require_backend_support, backend_support_note -from ._cli_utils import Status, dry_label, dry_status, omop_command, reject_reserved_schema, resolve_selection +from ._cli_utils import Status, dry_label, dry_status, omop_command, resolve_selection from .tables import ( TableCategory, - qualified_table_name, resolve_maintenance_tables, select_omop_tables, ) @@ -57,7 +57,6 @@ def analyze_tables( Runs on every ORM-managed table if both scope and table_names are omitted. """ - reject_reserved_schema(db_schema) if scope is not None and table_names is not None: raise RuntimeError("Use either `scope` or `table_names`, not both.") @@ -67,11 +66,7 @@ def analyze_tables( operation = "VACUUM ANALYZE" if vacuum else "ANALYZE" results: list[AnalyzeTableResult] = [] - connection_factory = ( - engine.connect().execution_options(isolation_level="AUTOCOMMIT") - if vacuum - else engine.connect() - ) + connection_factory = autocommit_connection(engine) if vacuum else engine.connect() with connection_factory as connection: for maintenance_table in selected_tables: @@ -88,7 +83,7 @@ def analyze_tables( continue if not dry_run: - backend.analyze_table(connection, maintenance_table.table_name, db_schema, vacuum=vacuum) + backend.analyze_table(connection, maintenance_table.table_name, vacuum=vacuum) results.append( AnalyzeTableResult( @@ -168,7 +163,6 @@ def truncate_tables( dry_run: bool = False, ) -> list[TruncateTableResult]: """Truncate selected ORM-managed tables. Raises if non-selected tables hold blocking FK references.""" - reject_reserved_schema(db_schema) if scope is not None and table_names is not None: raise RuntimeError("Use either `scope` or `table_names`, not both.") if scope is None and table_names is None: @@ -197,7 +191,7 @@ def truncate_tables( row_count = int( connection.exec_driver_sql( - f"SELECT COUNT(*) FROM {qualified_table_name(maintenance_table.table_name, db_schema)}" + f"SELECT COUNT(*) FROM {qualified(connection, maintenance_table.table_name)}" ).scalar_one() ) existing_tables.append(maintenance_table.table_name) @@ -224,7 +218,6 @@ def truncate_tables( backend.truncate_table_batch( connection, existing_tables, - db_schema, restart_identities=restart_identities, cascade=cascade, ) @@ -289,7 +282,6 @@ def reset_model_sequences( dry_run: bool = False, ) -> list[SequenceResetResult]: """Reset each owned sequence to MAX(pk_column) + 1 to prevent insert conflicts after bulk loads.""" - reject_reserved_schema(db_schema) backend = resolve_backend(engine) require_backend_support(backend, "find_sequence_name", "Sequence reset") inspector = sa.inspect(engine) @@ -302,7 +294,7 @@ def reset_model_sequences( continue sequence_name = backend.find_sequence_name( - connection, target.table_name, target.pk_column_name, db_schema + connection, target.table_name, target.pk_column_name ) if sequence_name is None: @@ -319,7 +311,7 @@ def reset_model_sequences( ) continue - fully_qualified = qualified_table_name(target.table_name, db_schema) + fully_qualified = qualified(connection, target.table_name) current_max = connection.execute( sa.text( f"SELECT COALESCE(MAX({target.pk_column_name}), 0) " diff --git a/omop_alchemy/maintenance/cli_vocab.py b/omop_alchemy/maintenance/cli_vocab.py index d336c10..2167cac 100644 --- a/omop_alchemy/maintenance/cli_vocab.py +++ b/omop_alchemy/maintenance/cli_vocab.py @@ -10,11 +10,11 @@ import sqlalchemy as sa import sqlalchemy.orm as so -import sqlalchemy.event as sae from sqlalchemy.exc import OperationalError import typer -from sqlalchemy.pool import NullPool -from orm_loader.backends import resolve_backend +from oa_configurator import ensure_schema +from orm_loader.backends import STAGING_SCHEMA, resolve_backend +from orm_loader.helpers import Base from orm_loader.tables.typing import CSVTableProtocol from rich.progress import ( BarColumn, @@ -39,17 +39,11 @@ Vocabulary, ) -from ._cli_utils import ( - ReservedSchema, - Status, - ensure_schema, - omop_command, - reject_reserved_schema, -) +from ._cli_utils import Status, omop_command from .cli_foreign_keys import manage_foreign_key_triggers from .cli_indexes import manage_indexes from .cli_tables import reset_model_sequences -from .tables import TableCategory, schema_adjusted_metadata, select_maintenance_tables +from .tables import TableCategory, select_maintenance_tables from .ui import ( console, render_error, @@ -261,13 +255,9 @@ def _create_missing_vocabulary_tables( if not missing_tables: return 0 - metadata, adjusted_tables = schema_adjusted_metadata( - vocab_tables, - db_schema=db_schema, - ) - metadata.create_all( + Base.metadata.create_all( bind=connection, - tables=[adjusted_tables[table.table_name] for table in missing_tables], + tables=[table.table for table in missing_tables], checkfirst=True, ) return len(missing_tables) @@ -276,6 +266,8 @@ def _create_missing_vocabulary_tables( def load_vocab_source( engine: sa.Engine, *, + vocab_engine: sa.Engine | None = None, + vocab_schema: str | None = None, source_path: str | Path, tables: list[str] | None = None, db_schema: str | None = None, @@ -299,7 +291,21 @@ def load_vocab_source( With bulk_mode (default on PostgreSQL), secondary indexes and FK triggers are toggled globally around the load for speed. Pass --no-bulk-mode when loading a single table to avoid the index drop/rebuild overhead. + + Parameters + ---------- + vocab_engine : sqlalchemy.Engine, optional + Engine to create vocabulary tables on, when ``vocab_connection`` names + a physically different server than ``engine``. Defaults to ``engine``. + CSV loading itself still runs entirely against ``engine``; this only + affects where vocab-role tables get created. + vocab_schema : str, optional + Schema vocab-role tables live in, for the table-existence check + against ``vocab_engine``. Defaults to ``db_schema``. """ + vocab_engine = vocab_engine if vocab_engine is not None else engine + vocab_schema = vocab_schema if vocab_schema is not None else db_schema + resolved_source_path = Path(source_path).expanduser().resolve() if not resolved_source_path.exists() or not resolved_source_path.is_dir(): raise RuntimeError(f"Athena source directory not found: {resolved_source_path}") @@ -333,31 +339,10 @@ def load_vocab_source( + ", ".join(sorted(missing)) ) - reject_reserved_schema(db_schema) - if not dry_run: ensure_schema(engine, db_schema) - ensure_schema(engine, ReservedSchema.STAGING) - - # NullPool: each session/connection is opened fresh and closed immediately after - # use. No stale pooled connections survive between tables, which prevents - # "connection in recovery mode" failures on subsequent tables after a heavy load. - load_engine = sa.create_engine(engine.url, poolclass=NullPool) - if db_schema is not None: - # NullPool discards the DBAPI connection on every commit, so a one-time - # SET search_path on the first checkout doesn't survive into the next - # checkout (e.g. after create_staging_table's commit). Re-apply on every - # new connection via an engine-level connect event so COPY and raw-cursor - # operations always target the right schema. The staging schema no longer - # needs to be on the path because orm-loader now qualifies staging - # table identifiers explicitly. - _quoted_schema = '"' + db_schema.replace('"', '""') + '"' - - @sae.listens_for(load_engine, "connect") - def _set_search_path(dbapi_conn, _record): - cur = dbapi_conn.cursor() - cur.execute(f"SET search_path TO {_quoted_schema}") - cur.close() + ensure_schema(engine, STAGING_SCHEMA) + ensure_schema(vocab_engine, vocab_schema) table_count = sum( 1 @@ -423,9 +408,9 @@ def _set_search_path(dbapi_conn, _record): ) if not dry_run: - with load_engine.connect() as pre_conn: + with vocab_engine.connect() as pre_conn: created_table_count = _create_missing_vocabulary_tables( - pre_conn, db_schema=db_schema + pre_conn, db_schema=vocab_schema ) pre_conn.commit() @@ -481,7 +466,7 @@ def _set_search_path(dbapi_conn, _record): _prev_attempt_was_crash = False for attempt in range(3): try: - with so.Session(load_engine) as session: + with so.Session(engine) as session: if ( _prev_attempt_was_crash and merge_strategy == "insert_if_empty" @@ -509,7 +494,7 @@ def _set_search_path(dbapi_conn, _record): index_strategy="keep" if _use_bulk_mode else "auto", chunksize=chunksize, merge_batch_size=merge_batch_size, - staging_schema=ReservedSchema.STAGING, + staging_schema=STAGING_SCHEMA, ) session.commit() break @@ -725,6 +710,8 @@ def _update_progress(event: VocabularyLoadProgress) -> None: report = load_vocab_source( engine, + vocab_engine=conn.vocab_engine, + vocab_schema=conn.vocab_schema, source_path=effective_athena_source, tables=tables or None, db_schema=conn.db_schema, diff --git a/omop_alchemy/maintenance/tables.py b/omop_alchemy/maintenance/tables.py index 04a5cf9..6cb8c2b 100644 --- a/omop_alchemy/maintenance/tables.py +++ b/omop_alchemy/maintenance/tables.py @@ -71,13 +71,6 @@ def has_single_integer_primary_key(self) -> bool: ) -def qualified_table_name(table_name: str, db_schema: str | None) -> str: - if db_schema: - quoted_schema = '"' + db_schema.replace('"', '""') + '"' - return f"{quoted_schema}.{table_name}" - return table_name - - def _mapped_cdm_table_classes() -> Iterable[type]: import omop_alchemy.cdm.model # noqa: F401 from orm_loader.helpers import Base @@ -257,23 +250,3 @@ def missing_maintenance_tables( for table in select_omop_tables(vocabulary_included=vocabulary_included) if not inspector.has_table(table.table_name, schema=db_schema) ] - - -def schema_adjusted_metadata( - tables: Iterable[MaintenanceTable], - *, - db_schema: str | None, -) -> tuple[sa.MetaData, dict[str, sa.Table]]: - metadata = sa.MetaData() - adjusted_tables: dict[str, sa.Table] = {} - - for maintenance_table in tables: - adjusted_tables[maintenance_table.table_name] = maintenance_table.table.to_metadata( - metadata, - schema=db_schema, # ty: ignore[invalid-argument-type] - referred_schema_fn=( - lambda _table, to_schema, _constraint, _referred_schema: to_schema - ), - ) - - return metadata, adjusted_tables diff --git a/pyproject.toml b/pyproject.toml index a4e93ec..8d2c84c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -91,3 +91,5 @@ packages = ["omop_alchemy"] cache-keys = [{ file = "pyproject.toml" }, { git = { commit = true, tags = true } }] [tool.pytest.ini_options] +addopts = "-m 'not db_dialect'" + diff --git a/tests/conftest.py b/tests/conftest.py index 76e8fe6..9770870 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,11 +4,13 @@ import pytest import sqlalchemy as sa from orm_loader.helpers import bootstrap +from oa_configurator.testing import isolated_test_database import sqlalchemy.orm as so from sqlalchemy.orm import Session, sessionmaker -from typing import Any, Dict, Tuple +from typing import Any, Dict, Iterator, Tuple +from omop_alchemy.config import OmopAlchemyConfig from omop_alchemy.maintenance.cli_vocab import _load_vocab_model_csv from omop_alchemy.cdm.model.clinical import Condition_Occurrence, Person from omop_alchemy.cdm.model.derived import Observation_Period @@ -24,6 +26,23 @@ ) +@pytest.fixture +def fresh_engine() -> Iterator[sa.Engine]: + """Fresh, empty, function-scoped SQLite engine. + + SQLite has no schema concept, so every role maps back to None, matching + the flat namespace every caller here has always assumed. + """ + with isolated_test_database( + OmopAlchemyConfig, + "test_cdm_db_sqlite", + dialect="sqlite", + future=True, + execution_options={"schema_translate_map": {None: None, "vocab": None, "results": None}}, + ) as db: + yield db.connection.engine + + ATHENA_LOAD_ORDER = [ Domain, Vocabulary, @@ -294,57 +313,84 @@ def _seed_basic_clinical_data(session: Session) -> None: @pytest.fixture(scope="session") -def engine(tmp_path_factory: pytest.TempPathFactory): +def engine(tmp_path_factory: pytest.TempPathFactory) -> Iterator[sa.Engine]: """ Session-scoped SQLite engine built from repo fixtures. - The database is created fresh for each test session, so it behaves - the same on the host and inside containers. + Resolves to a real tempfile (not ``:memory:``), matching this fixture's + own need to behave the same on the host and inside containers. """ - db_dir = tmp_path_factory.mktemp("omop-alchemy") - db_path = db_dir / "test.db" - engine = sa.create_engine( - f"sqlite:///{db_path}", + with isolated_test_database( + OmopAlchemyConfig, + "test_cdm_db_sqlite", + dialect="sqlite", future=True, echo=False, poolclass=sa.pool.StaticPool, connect_args={"check_same_thread": False, "timeout": 30}, - ) - - bootstrap(engine, create=True) - _load_fixture_vocabulary(engine, db_dir) - - with so.Session(engine, expire_on_commit=False) as seed_session: - _seed_basic_clinical_data(seed_session) + # SQLite has no schema concept, and this fixture always represented + # a single flat namespace: map every role back to None so the + # vocab/results-tagged tables land in the same place they always + # have here, unaffected by schema role tagging. + execution_options={"schema_translate_map": {None: None, "vocab": None, "results": None}}, + ) as db: + engine = db.connection.engine + bootstrap(engine, create=True) + _load_fixture_vocabulary(engine, tmp_path_factory.mktemp("omop-alchemy-fixtures")) + + with so.Session(engine, expire_on_commit=False) as seed_session: + _seed_basic_clinical_data(seed_session) - try: yield engine - finally: - engine.dispose() -@pytest.fixture(scope="session") -def pg_engine(): - """Session-scoped PostgreSQL engine for integration tests. +@pytest.fixture +def pg_db(request): + """Canonical isolated PostgreSQL test database (Phase 0 of the + schema_translate_map fix). - Resolves via OA_Configurator resource 'test_cdm_db' in ~/.config/omop/config.toml. + Resolves via OA_Configurator resource 'test_cdm_db_pg' in ~/.config/omop/config.toml. Run: omop-config configure omop_alchemy (answer Y when asked to configure test database). + + Everything a test does through ``pg_db.connection``/``pg_db.session`` + happens inside one transaction that's rolled back on exit: nothing + here is ever committed to the shared server, so concurrent test runs + can't collide and no manual cleanup is needed. + + New tests should use this directly rather than ``pg_engine``/``pg_session`` + below, which need a real, genuinely-committing ``Engine`` (for code that + calls ``.connect()``/``.begin()`` on what it's given) and are now thin + shims over this fixture's own connection. """ - from oa_configurator.pytest_plugin import ( - ensure_test_db_exists, - ensure_test_user_exists, - resolve_test_database, - ) + from oa_configurator.testing import isolated_test_database from omop_alchemy.config import OmopAlchemyConfig - url = resolve_test_database(OmopAlchemyConfig, "test_cdm_db") - ensure_test_user_exists(url) - ensure_test_db_exists(url) - engine = sa.create_engine(url, future=True) - try: - yield engine - finally: - engine.dispose() + with isolated_test_database(OmopAlchemyConfig, "test_cdm_db_pg", request=request) as db: + yield db + + +@pytest.fixture +def pg_engine(pg_db): + """Real, genuinely-committing PostgreSQL engine for tests that need + actual engine-building code paths (e.g. code that calls ``.connect()`` + or ``.begin()`` on what it's given, which a bare ``Connection`` can't + stand in for). + + A thin shim over ``pg_db``: reuses its already-resolved, test_only-checked + connection's own underlying ``Engine`` (``Connection.engine``) rather + than re-implementing resolution. Deliberately does not share pg_db's + rolled-back transaction: this fixture commits for real, isolated via + pg_session's own drop/recreate of the public schema instead. Function-scoped + (was session-scoped before this shim), since pg_db itself is + function-scoped and pytest fixtures can't depend on a narrower scope + than their own. + """ + return pg_db.connection.engine.execution_options( + # This fixture only ever creates/recreates the public schema below -- + # map every role back to None so vocab/results-tagged tables land + # there too, matching the single-schema setup this fixture provides. + schema_translate_map={None: None, "vocab": None, "results": None} + ) @pytest.fixture @@ -352,7 +398,15 @@ def pg_session(pg_engine): """ Function-scoped PostgreSQL session with a clean schema for each test. - Drops and recreates the public schema before each test to ensure full isolation. + Drops and recreates the public schema before each test to ensure full + isolation. Cannot move onto ``isolated_test_schema()``'s real, + uniquely-named schema: some tests request ``pg_session`` and + ``pg_engine`` together and rely on both pointing at the same physical + schema (``pg_engine`` itself has no schema of its own, only whatever + the connection's own default is). Confirmed by a real failure when + this migration was tried: ``test_load_vocab_postgres.py``'s + ``pg_session, pg_engine`` tests broke, since ``pg_engine``-only calls + then targeted an unbootstrapped schema. """ with pg_engine.connect() as conn: conn.execute(sa.text("DROP SCHEMA public CASCADE")) diff --git a/tests/test_analyze_tables.py b/tests/test_analyze_tables.py index 4465856..8f43451 100644 --- a/tests/test_analyze_tables.py +++ b/tests/test_analyze_tables.py @@ -1,4 +1,3 @@ -import sqlalchemy as sa import pytest from typer.testing import CliRunner @@ -9,18 +8,12 @@ runner = CliRunner() -def _engine(tmp_path): - """Create an isolated SQLite engine for analyze-table tests.""" - return sa.create_engine(f"sqlite:///{tmp_path / 'analyze.db'}", future=True) - - -def test_analyze_tables_runs_on_sqlite(tmp_path): +def test_analyze_tables_runs_on_sqlite(fresh_engine): """Analyze applies successfully on SQLite for selected OMOP tables.""" - engine = _engine(tmp_path) - create_missing_tables(engine, vocabulary_included=True) + create_missing_tables(fresh_engine, vocabulary_included=True) results = analyze_tables( - engine, + fresh_engine, scope=TableCategory.CLINICAL, dry_run=False, ) @@ -31,13 +24,11 @@ def test_analyze_tables_runs_on_sqlite(tmp_path): ) -def test_analyze_tables_rejects_vacuum_on_sqlite(tmp_path): +def test_analyze_tables_rejects_vacuum_on_sqlite(fresh_engine): """VACUUM ANALYZE is rejected on SQLite with a clear runtime error.""" - engine = _engine(tmp_path) - create_missing_tables(engine, vocabulary_included=True) + create_missing_tables(fresh_engine, vocabulary_included=True) with pytest.raises(RuntimeError) as exc_info: - analyze_tables(engine, scope=TableCategory.CLINICAL, vacuum=True) + analyze_tables(fresh_engine, scope=TableCategory.CLINICAL, vacuum=True) assert "not supported by the SQLite backend" in str(exc_info.value) - diff --git a/tests/test_backends_non_default_schema_postgres.py b/tests/test_backends_non_default_schema_postgres.py new file mode 100644 index 0000000..196e4fb --- /dev/null +++ b/tests/test_backends_non_default_schema_postgres.py @@ -0,0 +1,134 @@ +"""Non-default-schema Postgres coverage for the backends/ signature refactor (Phase 3.2). + +Every other maintenance-CLI test runs against the default schema, where +``schema_of(conn)`` returning ``None`` and the old ``db_schema=None`` +parameter are indistinguishable. The refactor that dropped explicit +``db_schema`` threading through ``backends/`` (deriving it internally via +``schema_of(conn)`` instead) could pass every existing test while still +being broken for a real non-default schema. This exercises a representative +subset of the refactored surface (FK trigger toggle, index create/drop, +full-text install, sequence reset) against a genuine non-default Postgres +schema, using ``pg_engine`` the same way ``test_db_schema_search_path_on_postgres`` +(``test_load_vocab_postgres.py``) already does. +""" + +from __future__ import annotations + +from typing import Iterator, NamedTuple + +import pytest +import sqlalchemy as sa + +from oa_configurator.testing import isolated_test_schema + +from omop_alchemy.maintenance._cli_utils import Status +from omop_alchemy.maintenance.cli_foreign_keys import ( + collect_foreign_key_trigger_status, + manage_foreign_key_triggers, +) +from omop_alchemy.maintenance.cli_fulltext import install_fulltext_columns +from omop_alchemy.maintenance.cli_indexes import manage_indexes +from omop_alchemy.maintenance.cli_schema_tables import create_missing_tables +from omop_alchemy.maintenance.cli_tables import reset_model_sequences + +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + + +class _Scoped(NamedTuple): + engine: sa.Engine + schema: str + + +@pytest.fixture() +def scoped(pg_engine: sa.Engine) -> Iterator[_Scoped]: + """``pg_engine`` scoped to a real, non-default, uniquely-named schema. + + Single-schema deployment: vocab/results fall back to the same schema as + everything else, matching ``ResolvedCDMDatabase``'s own fallback when + ``vocab_schema``/``results_schema`` aren't configured, the exact case + that made the vocab/results-role fix safe for unconfigured deployments. + """ + with isolated_test_schema(pg_engine, prefix="backends_non_default") as schema: + engine = pg_engine.execution_options( + schema_translate_map={None: schema, "vocab": schema, "results": schema} + ) + yield _Scoped(engine=engine, schema=schema) + + +def test_fk_trigger_toggle_targets_the_configured_schema(scoped: _Scoped) -> None: + create_missing_tables(scoped.engine, db_schema=scoped.schema, vocabulary_included=True) + + disabled = manage_foreign_key_triggers(scoped.engine, enable=False, db_schema=scoped.schema) + assert disabled + assert all(result.status == Status.APPLIED for result in disabled) + + status_after_disable = { + result.table_name: result + for result in collect_foreign_key_trigger_status(scoped.engine, db_schema=scoped.schema) + } + person_status = status_after_disable["person"] + assert person_status.enabled_trigger_count == 0 + assert person_status.disabled_trigger_count > 0 + + enabled = manage_foreign_key_triggers(scoped.engine, enable=True, db_schema=scoped.schema) + assert all(result.status == Status.APPLIED for result in enabled) + + status_after_enable = { + result.table_name: result + for result in collect_foreign_key_trigger_status(scoped.engine, db_schema=scoped.schema) + } + assert status_after_enable["person"].disabled_trigger_count == 0 + + +def test_index_disable_and_enable_targets_the_configured_schema(scoped: _Scoped) -> None: + create_missing_tables(scoped.engine, db_schema=scoped.schema, vocabulary_included=True) + + disabled = manage_indexes(scoped.engine, enable=False, db_schema=scoped.schema) + assert disabled + assert all(result.status in (Status.APPLIED, Status.SKIPPED) for result in disabled) + + inspector = sa.inspect(scoped.engine) + person_indexes_after_disable = { + idx["name"] for idx in inspector.get_indexes("person", schema=scoped.schema) + } + + enabled = manage_indexes(scoped.engine, enable=True, db_schema=scoped.schema) + assert all(result.status in (Status.APPLIED, Status.SKIPPED) for result in enabled) + + inspector = sa.inspect(scoped.engine) + person_indexes_after_enable = { + idx["name"] for idx in inspector.get_indexes("person", schema=scoped.schema) + } + assert person_indexes_after_enable != person_indexes_after_disable or person_indexes_after_enable + + +def test_fulltext_install_targets_the_configured_schema(scoped: _Scoped) -> None: + create_missing_tables(scoped.engine, db_schema=scoped.schema, vocabulary_included=True) + + results = install_fulltext_columns(scoped.engine, db_schema=scoped.schema) + assert results + assert all(result.status == Status.APPLIED for result in results) + + inspector = sa.inspect(scoped.engine) + for result in results: + columns = { + c["name"] for c in inspector.get_columns(result.table_name, schema=scoped.schema) + } + assert result.vector_column_name in columns + + +def test_sequence_reset_targets_the_configured_schema(scoped: _Scoped) -> None: + create_missing_tables(scoped.engine, db_schema=scoped.schema, vocabulary_included=True) + + results = { + r.table_name: r for r in reset_model_sequences(scoped.engine, db_schema=scoped.schema) + } + person_result = results["person"] + + assert person_result.status == Status.RESET + assert person_result.sequence_name is not None + assert person_result.next_value == 1 + + with scoped.engine.begin() as conn: + next_id = conn.execute(sa.text(f"SELECT nextval('{person_result.sequence_name}')")).scalar_one() + assert next_id == 1 diff --git a/tests/test_concept_groups.py b/tests/test_concept_groups.py index 452f606..857bb4d 100644 --- a/tests/test_concept_groups.py +++ b/tests/test_concept_groups.py @@ -228,20 +228,16 @@ def test_registered_identity_is_shared_across_engines(session): clear_concept_group_cache() -def test_unregistered_engines_do_not_share(session): +def test_unregistered_engines_do_not_share(session, fresh_engine): """Without an identity, each engine is its own scope. In-memory SQLite lands here deliberately: identical configuration, separate databases, so sharing would serve one database's concept sets for another. """ - other = sa.create_engine("sqlite://") - try: - with so.Session(other) as other_session: - assert concept_group_registry(session) is not concept_group_registry( - other_session - ) - finally: - other.dispose() + with so.Session(fresh_engine) as other_session: + assert concept_group_registry(session) is not concept_group_registry( + other_session + ) def test_connection_bound_sessions_share_their_engine_scope(session): diff --git a/tests/test_create_tables.py b/tests/test_create_tables.py index 4dd3484..4c7f4b5 100644 --- a/tests/test_create_tables.py +++ b/tests/test_create_tables.py @@ -3,27 +3,20 @@ from omop_alchemy.maintenance.cli_schema import collect_missing_tables, create_missing_tables -def _engine(tmp_path): - """Create an isolated SQLite engine for table-creation tests.""" - return sa.create_engine(f"sqlite:///{tmp_path / 'create_tables.db'}", future=True) - - -def test_collect_missing_tables_on_empty_database(tmp_path): +def test_collect_missing_tables_on_empty_database(fresh_engine): """An empty database reports core clinical and vocabulary tables as missing.""" - engine = _engine(tmp_path) - missing = collect_missing_tables(engine) + missing = collect_missing_tables(fresh_engine) table_names = {table.table_name for table in missing} assert "person" in table_names assert "concept" in table_names -def test_create_missing_tables_reports_blocked_tables_when_vocabulary_is_missing(tmp_path): +def test_create_missing_tables_reports_blocked_tables_when_vocabulary_is_missing(fresh_engine): """Non-vocabulary creation reports blocked tables when required vocab tables are excluded.""" - engine = _engine(tmp_path) - results = create_missing_tables(engine, vocabulary_included=False) + results = create_missing_tables(fresh_engine, vocabulary_included=False) - inspector = sa.inspect(engine) + inspector = sa.inspect(fresh_engine) assert results assert not inspector.has_table("concept") result_by_name = { @@ -34,27 +27,25 @@ def test_create_missing_tables_reports_blocked_tables_when_vocabulary_is_missing assert "concept" in result_by_name["person"].detail -def test_create_missing_tables_can_recreate_non_vocabulary_tables_when_dependencies_exist(tmp_path): +def test_create_missing_tables_can_recreate_non_vocabulary_tables_when_dependencies_exist(fresh_engine): """Previously dropped non-vocabulary tables can be recreated when dependencies are present.""" - engine = _engine(tmp_path) - create_missing_tables(engine, vocabulary_included=True) + create_missing_tables(fresh_engine, vocabulary_included=True) - with engine.begin() as connection: + with fresh_engine.begin() as connection: connection.exec_driver_sql("DROP TABLE cdm_source") - results = create_missing_tables(engine, vocabulary_included=False) + results = create_missing_tables(fresh_engine, vocabulary_included=False) - inspector = sa.inspect(engine) + inspector = sa.inspect(fresh_engine) assert any(result.table_name == "cdm_source" and result.status == "created" for result in results) assert inspector.has_table("cdm_source") assert inspector.has_table("concept") -def test_create_missing_tables_can_create_vocabulary(tmp_path): +def test_create_missing_tables_can_create_vocabulary(fresh_engine): """Including vocabulary creates both clinical and vocabulary tables.""" - engine = _engine(tmp_path) - create_missing_tables(engine, vocabulary_included=True) + create_missing_tables(fresh_engine, vocabulary_included=True) - inspector = sa.inspect(engine) + inspector = sa.inspect(fresh_engine) assert inspector.has_table("person") assert inspector.has_table("concept") diff --git a/tests/test_data_summary.py b/tests/test_data_summary.py index 0b4605d..82ace8f 100644 --- a/tests/test_data_summary.py +++ b/tests/test_data_summary.py @@ -4,45 +4,38 @@ from omop_alchemy.maintenance.cli_schema import collect_data_summary -def _engine(tmp_path): - return sa.create_engine(f"sqlite:///{tmp_path / 'data_summary.db'}", future=True) - - -def test_collect_data_summary_can_include_missing_tables(tmp_path): +def test_collect_data_summary_can_include_missing_tables(fresh_engine): """Test collect data summary can include missing tables.""" - engine = _engine(tmp_path) - results = collect_data_summary(engine, existing_only=False) + results = collect_data_summary(fresh_engine, existing_only=False) assert results assert any(result.exists is False for result in results) -def test_collect_data_summary_reports_row_counts(tmp_path): +def test_collect_data_summary_reports_row_counts(fresh_engine): """Test collect data summary reports row counts.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) - with engine.begin() as connection: + with fresh_engine.begin() as connection: connection.execute( sa.text("INSERT INTO location (location_id) VALUES (1)") ) results = { result.table_name: result - for result in collect_data_summary(engine, vocabulary_included=True) + for result in collect_data_summary(fresh_engine, vocabulary_included=True) } assert results["location"].exists is True assert results["location"].row_count == 1 -def test_collect_data_summary_excludes_vocabulary_by_default(tmp_path): +def test_collect_data_summary_excludes_vocabulary_by_default(fresh_engine): """Test collect data summary excludes vocabulary by default.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) table_names = { result.table_name - for result in collect_data_summary(engine) + for result in collect_data_summary(fresh_engine) } assert "person" in table_names assert "concept" not in table_names diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 12264a3..272c249 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -1,4 +1,3 @@ -import sqlalchemy as sa import pytest from typer.testing import CliRunner @@ -17,32 +16,26 @@ runner = CliRunner() -def _engine(tmp_path): - return sa.create_engine(f"sqlite:///{tmp_path / 'foreign_keys.db'}", future=True) - - -def test_collect_fk_info_finds_participating_tables(tmp_path): +def test_collect_fk_info_finds_participating_tables(fresh_engine): """Test _collect_fk_info finds participating tables.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) targets = { target.table_name: target - for target in _collect_fk_info(engine) + for target in _collect_fk_info(fresh_engine) } assert "person" in targets assert targets["person"].incoming_constraint_count > 0 -def test_manage_foreign_key_triggers_supports_dry_run(tmp_path): +def test_manage_foreign_key_triggers_supports_dry_run(fresh_engine): """Test manage foreign key triggers supports dry run.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) with pytest.raises(RuntimeError) as exc_info: manage_foreign_key_triggers( - engine, + fresh_engine, enable=False, dry_run=True, ) @@ -50,24 +43,22 @@ def test_manage_foreign_key_triggers_supports_dry_run(tmp_path): assert "not supported by the SQLite backend" in str(exc_info.value) -def test_collect_foreign_key_trigger_status_is_safe_on_sqlite(tmp_path): +def test_collect_foreign_key_trigger_status_is_safe_on_sqlite(fresh_engine): """Test collect foreign key trigger status is safe on sqlite.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) with pytest.raises(RuntimeError) as exc_info: - collect_foreign_key_trigger_status(engine) + collect_foreign_key_trigger_status(fresh_engine) assert "not supported by the SQLite backend" in str(exc_info.value) -def test_validate_foreign_key_constraints_is_safe_on_sqlite(tmp_path): +def test_validate_foreign_key_constraints_is_safe_on_sqlite(fresh_engine): """Test validate foreign key constraints is safe on sqlite.""" - engine = _engine(tmp_path) - create_missing_tables(engine) + create_missing_tables(fresh_engine) with pytest.raises(RuntimeError) as exc_info: - validate_foreign_key_constraints(engine) + validate_foreign_key_constraints(fresh_engine) assert "not supported by the SQLite backend" in str(exc_info.value) @@ -109,10 +100,10 @@ def name(self) -> str: def dialect(self) -> str: return "postgresql" - def analyze_table(self, conn, table_name, db_schema, *, vacuum=False) -> None: + def analyze_table(self, conn, table_name, *, vacuum=False) -> None: pass - def toggle_fk_triggers(self, conn, table_name, db_schema, *, enable: bool) -> None: + def toggle_fk_triggers(self, conn, table_name, *, enable: bool) -> None: action = "ENABLE" if enable else "DISABLE" conn.exec_driver_sql(f"ALTER TABLE {table_name} {action} TRIGGER ALL") diff --git a/tests/test_fulltext.py b/tests/test_fulltext.py index 9b08ef8..4a121de 100644 --- a/tests/test_fulltext.py +++ b/tests/test_fulltext.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import pytest +from sqlalchemy.dialects import postgresql from typer.testing import CliRunner from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig @@ -35,9 +36,14 @@ def __init__(self, rowcount: int | None = None): class _FakeConnection: - def __init__(self, *, rowcount: int = 7): + def __init__(self, *, rowcount: int = 7, db_schema: str | None = "public"): self.calls: list[tuple[str, str, dict[str, object] | None]] = [] self.rowcount = rowcount + self.dialect = postgresql.dialect() + self._db_schema = db_schema + + def get_execution_options(self) -> dict[str, object]: + return {"schema_translate_map": {None: self._db_schema}} def exec_driver_sql( self, @@ -70,8 +76,8 @@ def __exit__(self, exc_type, exc, tb) -> bool: class _FakeEngine: dialect = _FakeDialect() - def __init__(self, *, rowcount: int = 7): - self.connection = _FakeConnection(rowcount=rowcount) + def __init__(self, *, rowcount: int = 7, db_schema: str | None = "public"): + self.connection = _FakeConnection(rowcount=rowcount, db_schema=db_schema) def begin(self) -> _FakeBegin: return _FakeBegin(self.connection) @@ -122,7 +128,7 @@ def test_install_fulltext_columns_builds_postgresql_ddl_and_registers_metadata() assert all(result.status == "applied" for result in results) statements = [call[1] for call in engine.connection.calls] assert any( - 'ALTER TABLE "public"."concept" ADD COLUMN IF NOT EXISTS concept_name_tsvector tsvector' in statement + 'ALTER TABLE public.concept ADD COLUMN IF NOT EXISTS concept_name_tsvector tsvector' in statement for statement in statements ) assert any( @@ -147,8 +153,8 @@ def test_populate_fulltext_columns_issues_update_with_regconfig_and_row_counts() assert all(result.status == "applied" for result in results) assert [result.row_count for result in results] == [11, 11] execute_calls = [call for call in engine.connection.calls if call[0] == "execute"] - assert any('UPDATE "public"."concept"' in call[1] for call in execute_calls) - assert any("CAST(:regconfig AS regconfig)" in call[1] for call in execute_calls) + assert any('UPDATE public.concept' in call[1] for call in execute_calls) + assert any("CAST(:regconfig AS REGCONFIG)" in call[1] for call in execute_calls) assert all(call[2] == {"regconfig": "simple"} for call in execute_calls) _postgres.unregister_fulltext_metadata() @@ -167,9 +173,9 @@ def test_drop_fulltext_columns_drops_schema_objects_and_unregisters_metadata(): assert [result.action for result in results] == [FullTextAction.DROP, FullTextAction.DROP] assert all(result.status == "applied" for result in results) statements = [call[1] for call in engine.connection.calls] - assert any('DROP INDEX IF EXISTS "public"."idx_gin_concept_name_tsvector"' in statement for statement in statements) + assert any('DROP INDEX IF EXISTS public.idx_gin_concept_name_tsvector' in statement for statement in statements) assert any( - 'ALTER TABLE "public"."concept" DROP COLUMN IF EXISTS concept_name_tsvector' in statement + 'ALTER TABLE public.concept DROP COLUMN IF EXISTS concept_name_tsvector' in statement for statement in statements ) assert CONCEPT_NAME_TSVECTOR_COLUMN not in Concept.__table__.c @@ -183,9 +189,9 @@ def test_drop_fulltext_columns_drops_schema_objects_and_unregisters_metadata(): "drop_fulltext_columns", ], ) -def test_fulltext_management_requires_postgresql(tmp_path, fn_name): +def test_fulltext_management_requires_postgresql(fresh_engine, fn_name): """Fulltext management APIs reject non-PostgreSQL engines.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'fulltext.db'}", future=True) + engine = fresh_engine fn = { "install_fulltext_columns": install_fulltext_columns, "populate_fulltext_columns": populate_fulltext_columns, diff --git a/tests/test_indexes.py b/tests/test_indexes.py index db357b2..fea04d7 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -1,13 +1,14 @@ import pytest import sqlalchemy as sa from typer.testing import CliRunner -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Resolver, StackConfig, qualified +from oa_configurator.testing import DIALECT_PARAMS from omop_alchemy.backends.sqlite import SQLiteBackend from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY, omop_index_name from omop_alchemy.maintenance.cli import app from omop_alchemy.maintenance.cli_schema import create_missing_tables -from omop_alchemy.maintenance._cli_utils import ReservedSchema, Status, reject_reserved_schema +from omop_alchemy.maintenance._cli_utils import MAINTENANCE_SCHEMA, Status from omop_alchemy.maintenance.ui import render_index_summary from omop_alchemy.maintenance.cli_indexes import ( IndexManagementResult, @@ -37,16 +38,49 @@ CONCEPT_SYNONYM_NAME_LOWER_INDEX = "ix_concept_synonym_concept_synonym_name_lower" -def _fresh_engine(tmp_path): - db_path = tmp_path / "indexes.db" - engine = sa.create_engine(f"sqlite:///{db_path}", future=True) +@pytest.fixture(params=DIALECT_PARAMS) +def indexed_engine(request): + """Every OMOP table created, on both a real Postgres backend and + SQLite: index bookkeeping/capture/clustering is dialect-portable logic, + not SQLite-specific. Only the postgresql param ever requests + pg_session, so the sqlite param never needs a database. + + pg_session's own isolation only resets the public schema, never + MAINTENANCE_SCHEMA (a separate, fixed schema bookkeeping tables live + in), so the bookkeeping table is dropped here explicitly. Otherwise + both its rows *and its very existence* stay visible to every later + test, indefinitely, including tests that specifically assert it hasn't + been created yet. + """ + if request.param == "postgresql": + engine = request.getfixturevalue("pg_session").get_bind() + bookkeeping_schema = get_bookkeeping_schema(engine) + inspector = sa.inspect(engine) + if inspector.has_table(_DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema): + table_ref = qualified(engine, _DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema) + with engine.begin() as connection: + connection.exec_driver_sql(f"DROP TABLE {table_ref}") + else: + engine = request.getfixturevalue("fresh_engine") create_missing_tables(engine) return engine -def test_collect_index_targets_excludes_vocabulary_by_default(tmp_path): +@pytest.fixture +def sqlite_indexed_engine(fresh_engine): + """fresh_engine with every OMOP table already created. + + For the handful of tests that monkeypatch SQLiteBackend's own methods, + or assert SQLite-specific reflection limitations directly, since those + are dialect-specific by construction, not candidates for indexed_engine. + """ + create_missing_tables(fresh_engine) + return fresh_engine + + +def test_collect_index_targets_excludes_vocabulary_by_default(indexed_engine): """Test collect index targets excludes vocabulary by default.""" - engine = _fresh_engine(tmp_path) + engine = indexed_engine targets = { (target.table_name, target.index_name) for target in collect_index_targets(engine) @@ -59,7 +93,7 @@ def test_collect_index_targets_excludes_vocabulary_by_default(tmp_path): @pytest.mark.filterwarnings( "ignore:Skipped unsupported reflection of expression-based index:sqlalchemy.exc.SAWarning" ) -def test_collect_index_targets_can_include_vocabulary(tmp_path): +def test_collect_index_targets_can_include_vocabulary(indexed_engine): """Test collect index targets can include vocabulary. Notes @@ -71,7 +105,7 @@ def test_collect_index_targets_can_include_vocabulary(tmp_path): for coverage of the indexes themselves. """ - engine = _fresh_engine(tmp_path) + engine = indexed_engine targets = { (target.table_name, target.index_name) for target in collect_index_targets(engine, vocabulary_included=True) @@ -99,25 +133,18 @@ def test_orm_index_metadata_carries_cluster_configuration(): def test_schema_metadata_indexes_keys_match_unadjusted_indexes(): - """Test schema metadata indexes keys match unadjusted indexes. - - manage_indexes() looks up schema-adjusted indexes using names taken from - the original, unadjusted ORM tables. If a column's index name is resolved - implicitly (e.g. via `index=True` rather than an explicit omop_index()), - SQLAlchemy's naming convention embeds the schema into the generated name, - so the schema-adjusted copy gets a different name and the lookup misses. - """ + """Every table/index pair from ORM metadata is present in the result.""" tables = select_omop_tables(vocabulary_included=True) - indexes = _schema_metadata_indexes(tables, db_schema="public") + indexes = _schema_metadata_indexes(tables) for table in tables: for index in table.table.indexes: assert (table.table_name, str(index.name)) in indexes -def test_manage_indexes_disable_and_enable_on_sqlite(tmp_path): +def test_manage_indexes_disable_and_enable_on_sqlite(sqlite_indexed_engine): """Test manage indexes disable and enable on sqlite.""" - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine inspector = sa.inspect(engine) before = { @@ -162,17 +189,17 @@ def test_manage_indexes_disable_and_enable_on_sqlite(tmp_path): assert PERSON_GENDER_INDEX in after_enable -def test_manage_indexes_enable_analyzes_tables_with_new_indexes(tmp_path, monkeypatch): +def test_manage_indexes_enable_analyzes_tables_with_new_indexes(sqlite_indexed_engine, monkeypatch): """Test manage indexes enable analyzes tables with new indexes.""" - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine manage_indexes(engine, enable=False) analyzed_tables: list[str] = [] original_analyze = SQLiteBackend.analyze_table - def recording_analyze(self, conn, table_name, db_schema, *, vacuum=False): + def recording_analyze(self, conn, table_name, *, vacuum=False): analyzed_tables.append(table_name) - return original_analyze(self, conn, table_name, db_schema, vacuum=vacuum) + return original_analyze(self, conn, table_name, vacuum=vacuum) monkeypatch.setattr(SQLiteBackend, "analyze_table", recording_analyze) @@ -181,15 +208,15 @@ def recording_analyze(self, conn, table_name, db_schema, *, vacuum=False): assert "person" in analyzed_tables -def test_manage_indexes_enable_skips_analyze_when_nothing_created(tmp_path, monkeypatch): +def test_manage_indexes_enable_skips_analyze_when_nothing_created(sqlite_indexed_engine, monkeypatch): """Test manage indexes enable skips analyze when nothing created.""" - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine analyzed_tables: list[str] = [] monkeypatch.setattr( SQLiteBackend, "analyze_table", - lambda self, conn, table_name, db_schema, *, vacuum=False: analyzed_tables.append(table_name), + lambda self, conn, table_name, *, vacuum=False: analyzed_tables.append(table_name), ) # All ORM-defined indexes already exist on a freshly created schema, so @@ -202,7 +229,7 @@ def test_manage_indexes_enable_skips_analyze_when_nothing_created(tmp_path, monk @pytest.mark.filterwarnings( "ignore:Skipped unsupported reflection of expression-based index:sqlalchemy.exc.SAWarning" ) -def test_manage_indexes_enable_is_idempotent_for_expression_indexes(tmp_path): +def test_manage_indexes_enable_is_idempotent_for_expression_indexes(sqlite_indexed_engine): """Test manage indexes enable is idempotent for expression indexes. SQLite cannot reflect expression-based indexes (e.g. lower(concept_name)), @@ -211,7 +238,7 @@ def test_manage_indexes_enable_is_idempotent_for_expression_indexes(tmp_path): resulting duplicate-create attempt and must report it as skipped rather than falsely claiming the index was (re)created. """ - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine for _ in range(2): results = manage_indexes(engine, enable=True, vocabulary_included=True) @@ -232,7 +259,7 @@ def test_manage_indexes_enable_is_idempotent_for_expression_indexes(tmp_path): @pytest.mark.filterwarnings( "ignore:Skipped unsupported reflection of expression-based index:sqlalchemy.exc.SAWarning" ) -def test_manage_indexes_disable_drops_expression_indexes_on_sqlite(tmp_path): +def test_manage_indexes_disable_drops_expression_indexes_on_sqlite(sqlite_indexed_engine): """Test manage indexes disable drops expression indexes on sqlite. SQLite can't reflect expression-based indexes, so `disable` must not gate @@ -242,7 +269,7 @@ def test_manage_indexes_disable_drops_expression_indexes_on_sqlite(tmp_path): the same way. The index must actually be removed, and a result row must always be reported. """ - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine def _index_exists(name: str) -> bool: with engine.connect() as connection: @@ -275,9 +302,9 @@ def _index_exists(name: str) -> bool: @pytest.mark.filterwarnings( "ignore:Skipped unsupported reflection of expression-based index:sqlalchemy.exc.SAWarning" ) -def test_manage_indexes_disable_is_idempotent_on_sqlite(tmp_path): +def test_manage_indexes_disable_is_idempotent_on_sqlite(indexed_engine): """A second disable run should report already-absent indexes as skipped.""" - engine = _fresh_engine(tmp_path) + engine = indexed_engine first_results = manage_indexes(engine, enable=False, vocabulary_included=True) second_results = manage_indexes(engine, enable=False, vocabulary_included=True) @@ -301,7 +328,7 @@ def test_manage_indexes_disable_is_idempotent_on_sqlite(tmp_path): assert "already absent" in result.detail -def test_manage_indexes_enable_clusters_then_analyzes(tmp_path, monkeypatch): +def test_manage_indexes_enable_clusters_then_analyzes(sqlite_indexed_engine, monkeypatch): """Test manage indexes enable clusters then analyzes, even with nothing created. `indexes enable --cluster` must ANALYZE a table whenever it was clustered, @@ -310,14 +337,14 @@ def test_manage_indexes_enable_clusters_then_analyzes(tmp_path, monkeypatch): clustering so planner stats reflect the final physical layout -- matching the standalone `indexes cluster` command's order. """ - engine = _fresh_engine(tmp_path) + engine = sqlite_indexed_engine calls: list[str] = [] - def fake_cluster_table(self, conn, table_name, index_name, db_schema): + def fake_cluster_table(self, conn, table_name, index_name): calls.append(f"cluster:{table_name}") - def fake_analyze_table(self, conn, table_name, db_schema, *, vacuum=False): + def fake_analyze_table(self, conn, table_name, *, vacuum=False): calls.append(f"analyze:{table_name}") monkeypatch.setattr(SQLiteBackend, "cluster_table", fake_cluster_table) @@ -530,18 +557,21 @@ def test_describe_shape_conflict_mentions_reason(): # ── Reserved schema guard ──────────────────────────────────────────────────────── +# The check itself lives in oa_configurator (resolved via CDMDatabaseConfig.resolve(), +# see oa-configurator's own test_resolver.py::TestReservedSchemaCollision). This +# confirms OMOP_Alchemy's own MAINTENANCE_SCHEMA registration (_cli_utils.py, +# module import time) is actually picked up by it. -def test_reject_reserved_schema_rejects_staging_and_maintenance(): - with pytest.raises(RuntimeError): - reject_reserved_schema(ReservedSchema.STAGING.value) - with pytest.raises(RuntimeError): - reject_reserved_schema(ReservedSchema.MAINTENANCE.value) - - -def test_reject_reserved_schema_allows_ordinary_schema(): - reject_reserved_schema("public") - reject_reserved_schema(None) +def test_resolving_cdm_database_with_maintenance_schema_name_raises(): + cfg = StackConfig.for_session( + connections={"c": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, + databases={ + "default": CDMDatabaseConfig(connection="c", schema_name=MAINTENANCE_SCHEMA) + }, + ) + with pytest.raises(RuntimeError, match=f"{MAINTENANCE_SCHEMA!r}.*omop_alchemy"): + Resolver(cfg).resolve_database("default") # ── Foreign-named equivalent index reconciliation ──────────────────────────────── @@ -570,8 +600,8 @@ def _person_gender_result(results: list[IndexManagementResult]) -> IndexManageme return matches[0] -def test_collect_index_targets_reports_foreign_named_equivalent_index(tmp_path): - engine = _fresh_engine(tmp_path) +def test_collect_index_targets_reports_foreign_named_equivalent_index(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") targets = { @@ -583,8 +613,8 @@ def test_collect_index_targets_reports_foreign_named_equivalent_index(tmp_path): assert ("person", PERSON_GENDER_INDEX) not in targets -def test_manage_indexes_enable_skips_creation_when_foreign_equivalent_exists(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_enable_skips_creation_when_foreign_equivalent_exists(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") results = manage_indexes(engine, enable=True, cluster=False) @@ -600,8 +630,8 @@ def test_manage_indexes_enable_skips_creation_when_foreign_equivalent_exists(tmp assert PERSON_GENDER_INDEX not in index_names -def test_manage_indexes_disable_captures_and_drops_foreign_equivalent_index(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_disable_captures_and_drops_foreign_equivalent_index(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") results = manage_indexes(engine, enable=False) @@ -620,8 +650,8 @@ def test_manage_indexes_disable_captures_and_drops_foreign_equivalent_index(tmp_ assert bookkeeping[0]["index_name"] == "idx_gender" -def test_manage_indexes_enable_restores_captured_foreign_index(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_enable_restores_captured_foreign_index(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") manage_indexes(engine, enable=False) @@ -643,8 +673,8 @@ def test_manage_indexes_enable_restores_captured_foreign_index(tmp_path): assert _dropped_indexes_rows(engine) == [] -def test_manage_indexes_disable_enable_round_trip_is_idempotent_with_capture(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_disable_enable_round_trip_is_idempotent_with_capture(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") for _ in range(2): @@ -657,8 +687,8 @@ def test_manage_indexes_disable_enable_round_trip_is_idempotent_with_capture(tmp assert _dropped_indexes_rows(engine) == [] -def test_manage_indexes_dry_run_previews_foreign_equivalent_without_mutating(tmp_path): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_dry_run_previews_foreign_equivalent_without_mutating(indexed_engine): + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender") results = manage_indexes(engine, enable=True, dry_run=True, cluster=False) @@ -669,24 +699,24 @@ def test_manage_indexes_dry_run_previews_foreign_equivalent_without_mutating(tmp inspector = sa.inspect(engine) assert not inspector.has_table( - _DROPPED_INDEXES_TABLE_NAME, schema=get_bookkeeping_schema(SQLiteBackend()) + _DROPPED_INDEXES_TABLE_NAME, schema=get_bookkeeping_schema(engine) ) -def test_bookkeeping_table_not_created_when_nothing_to_capture(tmp_path): - engine = _fresh_engine(tmp_path) +def test_bookkeeping_table_not_created_when_nothing_to_capture(indexed_engine): + engine = indexed_engine manage_indexes(engine, enable=False) manage_indexes(engine, enable=True, cluster=False) inspector = sa.inspect(engine) assert not inspector.has_table( - _DROPPED_INDEXES_TABLE_NAME, schema=get_bookkeeping_schema(SQLiteBackend()) + _DROPPED_INDEXES_TABLE_NAME, schema=get_bookkeeping_schema(engine) ) -def test_manage_indexes_enable_cluster_uses_restored_physical_name(tmp_path, monkeypatch): - engine = _fresh_engine(tmp_path) +def test_manage_indexes_enable_cluster_uses_restored_physical_name(sqlite_indexed_engine, monkeypatch): + engine = sqlite_indexed_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {EPISODE_PERSON_INDEX}") @@ -698,14 +728,14 @@ def test_manage_indexes_enable_cluster_uses_restored_physical_name(tmp_path, mon calls: list[tuple[str, str]] = [] - def fake_cluster_table(self, conn, table_name, index_name, db_schema): + def fake_cluster_table(self, conn, table_name, index_name): calls.append((table_name, index_name)) monkeypatch.setattr(SQLiteBackend, "cluster_table", fake_cluster_table) monkeypatch.setattr( SQLiteBackend, "analyze_table", - lambda self, conn, table_name, db_schema, *, vacuum=False: None, + lambda self, conn, table_name, *, vacuum=False: None, ) manage_indexes(engine, enable=True, cluster=True) @@ -714,9 +744,9 @@ def fake_cluster_table(self, conn, table_name, index_name, db_schema): def test_manage_indexes_disable_warns_and_leaves_unsupported_foreign_index_in_place( - tmp_path, monkeypatch + indexed_engine, monkeypatch ): - engine = _fresh_engine(tmp_path) + engine = indexed_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") @@ -756,13 +786,14 @@ def fake_get_indexes(self, table_name, schema=None, **kw): def _dropped_indexes_rows(engine: sa.Engine) -> list[dict[str, object]]: - bookkeeping_schema = get_bookkeeping_schema(SQLiteBackend()) + bookkeeping_schema = get_bookkeeping_schema(engine) inspector = sa.inspect(engine) if not inspector.has_table(_DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema): return [] + table_ref = qualified(engine, _DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema) with engine.connect() as connection: rows = connection.exec_driver_sql( - f"SELECT table_name, index_name FROM {_DROPPED_INDEXES_TABLE_NAME}" + f"SELECT table_name, index_name FROM {table_ref}" ).mappings().all() return [dict(row) for row in rows] @@ -833,12 +864,12 @@ def test_is_plain_index_false_for_constraint_backed_index(): assert "constraint" in _describe_shape_conflict(constraint_backed) -def test_manage_indexes_disable_second_run_without_enable_degrades_to_warning(tmp_path): +def test_manage_indexes_disable_second_run_without_enable_degrades_to_warning(indexed_engine): """A second disable run, without an intervening enable, that finds a different foreign index than the one already captured must not crash. It must leave the second index in place and report a warning, since the bookkeeping table can only track one pending capture per table/column-set.""" - engine = _fresh_engine(tmp_path) + engine = indexed_engine _replace_with_foreign_index(engine, foreign_name="idx_gender_v1") first = manage_indexes(engine, enable=False) @@ -863,21 +894,20 @@ def test_manage_indexes_disable_second_run_without_enable_degrades_to_warning(tm assert restored.index_name == "idx_gender_v1" -def test_record_captured_index_scopes_by_db_schema(tmp_path): +def test_record_captured_index_scopes_by_db_schema(indexed_engine): """Two different schemas capturing an equivalent foreign index for a same-named table/column-set must not collide. Each capture is independent and neither should be rejected by the other's bookkeeping row.""" - engine = _fresh_engine(tmp_path) - backend = SQLiteBackend() + engine = indexed_engine with engine.begin() as connection: captured_a = _record_captured_index( - connection, backend, + connection, table_name="person", db_schema="site_a", index_name="idx_a", column_names=("gender_concept_id",), unique=False, ) captured_b = _record_captured_index( - connection, backend, + connection, table_name="person", db_schema="site_b", index_name="idx_b", column_names=("gender_concept_id",), unique=False, ) @@ -890,7 +920,7 @@ def test_record_captured_index_scopes_by_db_schema(tmp_path): # without_enable_degrades_to_warning covers end-to-end). with engine.begin() as connection: captured_a_again = _record_captured_index( - connection, backend, + connection, table_name="person", db_schema="site_a", index_name="idx_a_v2", column_names=("gender_concept_id",), unique=False, ) diff --git a/tests/test_load_vocab.py b/tests/test_load_vocab.py index 05ecda8..db2bc3e 100644 --- a/tests/test_load_vocab.py +++ b/tests/test_load_vocab.py @@ -1,6 +1,5 @@ import pytest from orm_loader.helpers import bootstrap -import sqlalchemy as sa from sqlalchemy.orm import sessionmaker from omop_alchemy.cdm.model.vocabulary import ( @@ -14,21 +13,15 @@ from tests.conftest import ATHENA_LOAD_ORDER, _ATHENA_FIXTURE_DATA, _write_fixture_csv -@pytest.fixture(scope="session") -def connection(): +@pytest.fixture +def connection(fresh_engine): """ In-memory SQLite database for tests. """ - engine = sa.create_engine( - "sqlite+pysqlite:///:memory:", - future=True, - ) - - connection = engine.connect() + connection = fresh_engine.connect() bootstrap(connection, create=True) # type: ignore[arg-type] yield connection connection.close() - engine.dispose() @pytest.fixture @@ -46,15 +39,15 @@ def db_session(connection): session.close() -@pytest.fixture(scope="session") -def athena_vocab(connection, tmp_path_factory): +@pytest.fixture +def athena_vocab(connection, tmp_path): """ Load the minimal Athena vocabulary fixture using the real ORM CSV loader. Writes in-memory fixture data to a temp directory so no static CSV files on disk are required. """ - base_path: Path = tmp_path_factory.mktemp("athena_vocab") + base_path: Path = tmp_path Session = sessionmaker(bind=connection, future=True) session = Session() diff --git a/tests/test_load_vocab_postgres.py b/tests/test_load_vocab_postgres.py index 7e33727..e0e9ac3 100644 --- a/tests/test_load_vocab_postgres.py +++ b/tests/test_load_vocab_postgres.py @@ -4,8 +4,9 @@ These tests require a running PostgreSQL container. Start one with: docker compose -f tests/docker-compose.yaml up -d -Then run: - pytest -m postgres +Excluded from the default `pytest` invocation (addopts = "-m 'not +db_dialect'", see oa_configurator.testing). Run explicitly: + pytest -m postgresql """ from pathlib import Path @@ -21,6 +22,8 @@ ) from tests.conftest import _ATHENA_FIXTURE_DATA, _write_fixture_csv +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + def _copy_fixture_source(base_dir: Path) -> Path: """Write the shared in-memory Athena fixture set into an isolated per-test source dir.""" @@ -75,7 +78,6 @@ def _make_concept_source( # --------------------------------------------------------------------------- -@pytest.mark.requires_database("test_cdm_db") def test_end_to_end_vocab_load_on_postgres(pg_session, pg_engine, tmp_path): """load_vocab_source() completes end-to-end on real Postgres via orm-loader>=0.4.0.""" source_path = _copy_fixture_source(tmp_path) @@ -89,7 +91,6 @@ def test_end_to_end_vocab_load_on_postgres(pg_session, pg_engine, tmp_path): assert count == 7 -@pytest.mark.requires_database("test_cdm_db") def test_default_quote_mode_preserves_literal_quotes_on_postgres( pg_session, pg_engine, tmp_path ): @@ -133,7 +134,6 @@ def test_default_quote_mode_preserves_literal_quotes_on_postgres( assert concept_name == quoted_name -@pytest.mark.requires_database("test_cdm_db") def test_explicit_csv_quote_mode_strips_quotes_on_postgres( pg_session, pg_engine, tmp_path ): @@ -173,7 +173,6 @@ def test_explicit_csv_quote_mode_strips_quotes_on_postgres( assert concept_name == long_name -@pytest.mark.requires_database("test_cdm_db") def test_load_vocab_model_csv_on_postgres(pg_session, tmp_path): """ _load_vocab_model_csv loads data correctly on a real PostgreSQL session. @@ -197,7 +196,6 @@ def test_load_vocab_model_csv_on_postgres(pg_session, tmp_path): assert count == 7 -@pytest.mark.requires_database("test_cdm_db") def test_replace_strategy_overwrites_matching_and_preserves_absent_rows( pg_session, pg_engine, @@ -235,7 +233,6 @@ def test_replace_strategy_overwrites_matching_and_preserves_absent_rows( assert names[source_absent_id] == "preserved" -@pytest.mark.requires_database("test_cdm_db") def test_upsert_strategy_is_non_destructive(pg_session, pg_engine, tmp_path): """merge_strategy='upsert' preserves existing rows on second load with same PKs.""" concept_id = 99998 @@ -258,11 +255,16 @@ def test_upsert_strategy_is_non_destructive(pg_session, pg_engine, tmp_path): ) -@pytest.mark.requires_database("test_cdm_db") def test_db_schema_search_path_on_postgres(pg_engine, tmp_path): """ load_vocab_source with db_schema creates vocabulary tables in the requested PostgreSQL schema and loads data into them correctly. + + schema_translate_map, not db_schema alone, is what actually routes + ORM-managed table creation: a real deployment sets it once, at engine + construction (ResolvedCDMDatabase.create_engine()), not per call. This + scopes it here the same way, matching orm-loader's own + test_schema_translate_map.py regression test. """ schema = "VocabTest" source_path = _copy_fixture_source(tmp_path) @@ -273,9 +275,16 @@ def test_db_schema_search_path_on_postgres(pg_engine, tmp_path): conn.execute(sa.text(f"CREATE SCHEMA {quoted_schema}")) conn.commit() + # A single-schema deployment: vocab/results fall back to the same + # schema as everything else, matching ResolvedCDMDatabase's own default + # fallback behaviour when vocab_schema/results_schema aren't configured. + scoped_engine = pg_engine.execution_options( + schema_translate_map={None: schema, "vocab": schema, "results": schema} + ) + try: report = load_vocab_source( - pg_engine, + scoped_engine, source_path=source_path, db_schema=schema, ) @@ -298,7 +307,6 @@ def test_db_schema_search_path_on_postgres(pg_engine, tmp_path): conn.commit() -@pytest.mark.requires_database("test_cdm_db") def test_postgres_catalog_queries_accept_explicit_schema(pg_engine): """Schema-qualified catalog checks must bind cleanly with psycopg/PostgreSQL.""" backend = PostgresBackend() @@ -307,12 +315,10 @@ def test_postgres_catalog_queries_accept_explicit_schema(pg_engine): disabled, enabled = backend.get_fk_trigger_counts( connection, "concept", - "public", ) clustered_index = backend.get_clustered_index_name( connection, "concept", - "public", ) assert disabled >= 0 diff --git a/tests/test_load_vocab_source.py b/tests/test_load_vocab_source.py index 01d55a6..33dfb35 100644 --- a/tests/test_load_vocab_source.py +++ b/tests/test_load_vocab_source.py @@ -59,13 +59,12 @@ def _write_csv_with_size(source_path: Path, table_name: str, size_bytes: int) -> def test_load_vocab_source_on_sqlite_creates_tables_and_reports_loaded_results( + fresh_engine, monkeypatch, tmp_path, ): """Test load vocab source on sqlite creates tables and reports loaded results.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) loaded_tables: list[tuple[str, str, str, Path]] = [] @@ -114,11 +113,9 @@ def fake_load_vocab_model_csv( assert inspector.has_table("concept") -def test_load_vocab_source_requires_full_required_athena_fixture(tmp_path): +def test_load_vocab_source_requires_full_required_athena_fixture(fresh_engine, tmp_path): """Test load vocab source requires full required athena fixture.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_missing_required.db'}", future=True - ) + engine = fresh_engine # Build a source with only a subset of required models to trigger the missing-files error. partial_source = tmp_path / "partial_athena" @@ -146,11 +143,9 @@ def test_drug_strength_model_matches_athena_vocabulary_shape(): assert "end_datetime" not in column_names -def test_load_vocab_source_dry_run_does_not_create_tables(tmp_path): +def test_load_vocab_source_dry_run_does_not_create_tables(fresh_engine, tmp_path): """Test load vocab source dry run does not create tables.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_dry_run.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) report = load_vocab_source( @@ -194,6 +189,8 @@ def test_load_vocab_source_cli_uses_configured_athena_source(monkeypatch, tmp_pa def fake_load_vocab_source( engine: object, *, + vocab_engine: object = None, # noqa: ARG001 + vocab_schema: str | None = None, # noqa: ARG001 source_path: str | Path, tables: list[str] | None = None, # noqa: ARG001 db_schema: str | None = None, @@ -241,11 +238,9 @@ def fake_load_vocab_source( assert "load-vocab-source" in result.stdout -def test_load_vocab_model_csv_passes_quote_mode(monkeypatch, tmp_path): +def test_load_vocab_model_csv_passes_quote_mode(fresh_engine, monkeypatch, tmp_path): """Test load vocab model csv passes quote mode.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_quote_mode.db'}", future=True - ) + engine = fresh_engine class FakeModel: __tablename__ = "concept" @@ -294,11 +289,9 @@ def fake_load_csv( assert calls["quote_mode"] == "literal" -def test_load_vocab_source_loads_in_fk_dependency_order(monkeypatch, tmp_path): +def test_load_vocab_source_loads_in_fk_dependency_order(fresh_engine, monkeypatch, tmp_path): """Tables must be loaded in REQUIRED_VOCAB_MODELS order to respect FK dependencies.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_order.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) # Give domain a tiny file and concept_class a large one — if size-sorting were still in place @@ -335,11 +328,9 @@ def fake_load_vocab_model_csv( assert loaded_order[: len(expected_order)] == expected_order -def test_load_vocab_source_reports_weighted_progress(monkeypatch, tmp_path): +def test_load_vocab_source_reports_weighted_progress(fresh_engine, monkeypatch, tmp_path): """Test load vocab source reports weighted progress.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_progress.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) _write_csv_with_size(source_path, "domain", 10) @@ -379,11 +370,9 @@ def fake_load_vocab_model_csv( assert percents == sorted(percents) -def test_load_vocab_source_wraps_failed_table_load(monkeypatch, tmp_path): +def test_load_vocab_source_wraps_failed_table_load(fresh_engine, monkeypatch, tmp_path): """Test load vocab source wraps failed table load.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_error.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) def fake_load_vocab_model_csv( @@ -423,11 +412,9 @@ def fake_load_vocab_model_csv( assert "value too long for type character varying(255)" in message -def test_load_vocab_model_csv_retries_missing_staging_table(monkeypatch, tmp_path): +def test_load_vocab_model_csv_retries_missing_staging_table(fresh_engine, monkeypatch, tmp_path): """Test load vocab model csv retries missing staging table.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_retry.db'}", future=True - ) + engine = fresh_engine class FakeModel: __tablename__ = "drug_strength" @@ -527,11 +514,9 @@ def fail_load_vocab_source(*args, **kwargs): assert "value too long for type character varying(255)" in result.stdout -def test_load_vocab_source_defaults_to_by_delimiter_quote_mode(monkeypatch, tmp_path): +def test_load_vocab_source_defaults_to_by_delimiter_quote_mode(fresh_engine, monkeypatch, tmp_path): """Tab-delimited Athena quotes are literal data unless explicitly overridden.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'quote_mode_default.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) received_quote_modes: list[str] = [] @@ -565,20 +550,18 @@ def fake_load_vocab_model_csv( assert "csv" not in received_quote_modes -def test_load_vocab_source_tables_unknown_name_raises_runtime_error(tmp_path): +def test_load_vocab_source_tables_unknown_name_raises_runtime_error(fresh_engine, tmp_path): """Unknown table name in tables= is rejected before any DB connection.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'tables_unknown.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) with pytest.raises(RuntimeError, match="Unknown vocabulary table"): load_vocab_source(engine, source_path=source_path, tables=["not_a_table"]) -def test_load_vocab_source_tables_single_loads_only_that_table(monkeypatch, tmp_path): +def test_load_vocab_source_tables_single_loads_only_that_table(fresh_engine, monkeypatch, tmp_path): """tables=['concept'] loads only concept and skips every other table.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'tables_single.db'}", future=True) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) loaded_tables: list[str] = [] @@ -609,11 +592,9 @@ def fake_load_vocab_model_csv( assert result_names == {"concept"} -def test_load_vocab_source_tables_multiple_loads_exactly_those(monkeypatch, tmp_path): +def test_load_vocab_source_tables_multiple_loads_exactly_those(fresh_engine, monkeypatch, tmp_path): """tables=['concept', 'vocabulary'] loads exactly those two tables.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'tables_multiple.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) loaded_tables: list[str] = [] @@ -642,11 +623,9 @@ def fake_load_vocab_model_csv( assert set(loaded_tables) == {"concept", "vocabulary"} -def test_load_vocab_source_tables_skips_required_files_preflight(tmp_path): +def test_load_vocab_source_tables_skips_required_files_preflight(fresh_engine, tmp_path): """tables= skips the all-required-files gate even when most CSVs are absent.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'tables_preflight.db'}", future=True - ) + engine = fresh_engine # Only concept.csv present — would fail the all-required-files check without tables=. source_path = tmp_path / "sparse" @@ -663,11 +642,9 @@ def test_load_vocab_source_tables_skips_required_files_preflight(tmp_path): assert result_names == {"concept"} -def test_load_vocab_source_tables_missing_csv_raises_runtime_error(tmp_path): +def test_load_vocab_source_tables_missing_csv_raises_runtime_error(fresh_engine, tmp_path): """Explicitly named table whose CSV is absent raises RuntimeError, not a silent skip.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'tables_missing_csv.db'}", future=True - ) + engine = fresh_engine source_path = tmp_path / "empty_source" source_path.mkdir() @@ -677,13 +654,11 @@ def test_load_vocab_source_tables_missing_csv_raises_runtime_error(tmp_path): load_vocab_source(engine, source_path=source_path, tables=["concept"]) -def test_load_vocab_source_bulk_mode_surfaces_index_warnings(monkeypatch, tmp_path): +def test_load_vocab_source_bulk_mode_surfaces_index_warnings(fresh_engine, monkeypatch, tmp_path): """A foreign index that manage_indexes(enable=False) leaves in place (status=warning) during the bulk-mode disable step must be surfaced on the returned report, not silently discarded -- this is the only call site that inspects those results.""" - engine = sa.create_engine( - f"sqlite:///{tmp_path / 'load_vocab_source_bulk.db'}", future=True - ) + engine = fresh_engine source_path = _build_required_athena_source(tmp_path) # Force the bulk-mode gate (which requires a PostgreSQL engine) without needing diff --git a/tests/test_oncology_episode.py b/tests/test_oncology_episode.py index 2a03156..80d4fbd 100644 --- a/tests/test_oncology_episode.py +++ b/tests/test_oncology_episode.py @@ -104,9 +104,9 @@ def test_oncology_episode_does_not_expose_generic_drug_episode_interface(): @pytest.fixture -def oncology_session(tmp_path) -> Iterator[so.Session]: +def oncology_session(fresh_engine) -> Iterator[so.Session]: """Committed oncology graph so vocabulary-cache sessions see its closure.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'oncology.db'}") + engine = fresh_engine bootstrap(engine, create=True) rt_concept_id = 9_100_001 diff --git a/tests/test_schema_doctor.py b/tests/test_schema_doctor.py index f7a1e6f..9b76c54 100644 --- a/tests/test_schema_doctor.py +++ b/tests/test_schema_doctor.py @@ -5,9 +5,10 @@ def test_doctor_uses_borrowed_engine_without_resolving_config_or_disposing( + fresh_engine, monkeypatch, ) -> None: - engine = sa.create_engine("sqlite://") + engine = fresh_engine disposed_engines: list[sa.engine.Engine] = [] inspected: dict[str, object] = {} original_dispose = sa.engine.Engine.dispose @@ -43,7 +44,7 @@ def track_dispose(self, *args, **kwargs): vocabulary_included=False, ) - assert report.info.engine_url == "sqlite://" + assert report.info.engine_url == str(engine.url) assert report.info.backend == "sqlite" assert report.info.db_schema == "analytics" assert report.info.resource_name == "manual_cdm" diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index d44d68e..1167d51 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -1,7 +1,9 @@ -import sqlalchemy as sa +import pytest +from oa_configurator.testing import DIALECT_PARAMS from omop_alchemy.backends.sqlite import SQLiteBackend from omop_alchemy.cdm.base.indexing import omop_index_name +from omop_alchemy.maintenance.cli_indexes import manage_indexes from omop_alchemy.maintenance.cli_schema import create_missing_tables from omop_alchemy.maintenance.cli_schema_reconcile import is_blocking_issue, reconcile_schema @@ -9,13 +11,51 @@ EPISODE_PERSON_INDEX = omop_index_name("episode", "person_id") -def _fresh_engine(tmp_path): - db_path = tmp_path / "reconcile.db" - engine = sa.create_engine(f"sqlite:///{db_path}", future=True) +@pytest.fixture(params=DIALECT_PARAMS) +def reconcile_engine(request): + """Every OMOP table created and indexed/clustered, on both a real + Postgres backend and SQLite: index-rename detection is dialect-portable + logic, not SQLite-specific, so it's genuinely worth exercising against + both. Only the postgresql param ever requests pg_session, so the + sqlite param never needs a database. + + DIALECT_PARAMS carries each dialect's own mark plus `forked` directly + on the param value, so this still works correctly even though + request.getfixturevalue("pg_session") is a dynamic, runtime lookup + invisible to pytest's collection-time fixturenames computation (the + usual pg_db-in-fixturenames auto-detection can't see it). + + manage_indexes(enable=True) matters here specifically on Postgres: + create_missing_tables() alone creates tables and their indexes, but + never physically CLUSTERs them, so a fresh Postgres database reports + genuine drift (cluster status MISSING) without this step. On SQLite, + where CLUSTER doesn't exist, this call is a harmless no-op on the + clustering half and just re-confirms indexes already exist. + """ + if request.param == "postgresql": + engine = request.getfixturevalue("pg_session").get_bind() + else: + engine = request.getfixturevalue("fresh_engine") create_missing_tables(engine) + manage_indexes(engine, enable=True) return engine +@pytest.fixture +def fresh_reconcile_engine(fresh_engine): + """fresh_engine with every OMOP table already created. + + SQLite-only, unlike reconcile_engine above: the tests using this + fixture monkeypatch SQLiteBackend.get_clustered_index_name directly to + simulate a physical CLUSTER state, since SQLite has no real clustering + to test against at all (CLUSTER is a genuine Postgres-only physical + operation). Parametrizing these onto Postgres would need a real CLUSTER + call, not a mock swap, so they stay a separate, SQLite-specific fixture. + """ + create_missing_tables(fresh_engine) + return fresh_engine + + def _person_gender_issues(report): return [ issue @@ -26,8 +66,8 @@ def _person_gender_issues(report): ] -def test_reconcile_schema_reports_no_drift_on_fresh_database(tmp_path): - engine = _fresh_engine(tmp_path) +def test_reconcile_schema_reports_no_drift_on_fresh_database(reconcile_engine): + engine = reconcile_engine report = reconcile_schema(engine) person_result = next(r for r in report.table_results if r.table_name == "person") @@ -35,8 +75,8 @@ def test_reconcile_schema_reports_no_drift_on_fresh_database(tmp_path): assert person_result.issue_count == 0 -def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(tmp_path): - engine = _fresh_engine(tmp_path) +def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(reconcile_engine): + engine = reconcile_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") connection.exec_driver_sql("CREATE INDEX idx_gender ON person (gender_concept_id)") @@ -51,8 +91,8 @@ def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(tmp assert issue.actual == "idx_gender" -def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(tmp_path): - engine = _fresh_engine(tmp_path) +def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(reconcile_engine): + engine = reconcile_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") connection.exec_driver_sql("CREATE INDEX idx_gender ON person (gender_concept_id)") @@ -83,11 +123,11 @@ def test_is_blocking_issue_excludes_renamed_only(): assert is_blocking_issue(missing) is True -def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_index(tmp_path, monkeypatch): +def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_index(fresh_reconcile_engine, monkeypatch): """A table physically clustered on a foreign-named equivalent of the ORM's cluster index (e.g. captured/restored under its original name by manage_indexes()) must report a 'renamed' cluster issue, not 'mismatch'.""" - engine = _fresh_engine(tmp_path) + engine = fresh_reconcile_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {EPISODE_PERSON_INDEX}") connection.exec_driver_sql("CREATE INDEX idx_episode_person ON episode (person_id)") @@ -95,7 +135,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_inde monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name, db_schema: ( + lambda self, conn, table_name: ( "idx_episode_person" if table_name == "episode" else None ), ) @@ -114,15 +154,15 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_inde assert episode_result.status == "matched" -def test_reconcile_schema_cluster_check_still_reports_real_mismatch(tmp_path, monkeypatch): +def test_reconcile_schema_cluster_check_still_reports_real_mismatch(fresh_reconcile_engine, monkeypatch): """A genuinely different physical cluster state (not just a foreign-named equivalent) must still be reported as drift.""" - engine = _fresh_engine(tmp_path) + engine = fresh_reconcile_engine monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name, db_schema: ( + lambda self, conn, table_name: ( "some_unrelated_index" if table_name == "episode" else None ), ) @@ -139,7 +179,7 @@ def test_reconcile_schema_cluster_check_still_reports_real_mismatch(tmp_path, mo assert episode_result.status == "drifted" -def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_target(tmp_path, monkeypatch): +def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_target(fresh_reconcile_engine, monkeypatch): """person's cluster target is the primary key's own index ("pk_person"), not a declared secondary index, unlike episode. The official OHDSI CDM DDL always clusters such tables on a separate, non-unique index instead @@ -149,14 +189,14 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_tar equivalence check assuming the PK's own uniqueness applies to whatever physically serves as the cluster index, and not being shared with the general index-diffing pass).""" - engine = _fresh_engine(tmp_path) + engine = fresh_reconcile_engine with engine.begin() as connection: connection.exec_driver_sql("CREATE INDEX idx_person_id ON person (person_id)") monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name, db_schema: ( + lambda self, conn, table_name: ( "idx_person_id" if table_name == "person" else None ), ) diff --git a/tests/test_truncate_tables.py b/tests/test_truncate_tables.py index 9982d00..5c4f3c7 100644 --- a/tests/test_truncate_tables.py +++ b/tests/test_truncate_tables.py @@ -1,5 +1,4 @@ import importlib -import sqlalchemy as sa import pytest from typer.testing import CliRunner from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig @@ -14,9 +13,9 @@ truncate_tables_module = importlib.import_module("omop_alchemy.maintenance.cli_tables") -def test_truncate_tables_requires_postgresql(tmp_path): +def test_truncate_tables_requires_postgresql(fresh_engine): """Test truncate tables requires postgresql.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'truncate.db'}", future=True) + engine = fresh_engine with pytest.raises(RuntimeError) as exc_info: truncate_tables(engine, scope=TableCategory.CLINICAL, dry_run=True) @@ -24,9 +23,9 @@ def test_truncate_tables_requires_postgresql(tmp_path): assert "not supported by the SQLite backend" in str(exc_info.value) -def test_truncate_tables_reports_blocking_foreign_key_references(monkeypatch, tmp_path): +def test_truncate_tables_reports_blocking_foreign_key_references(monkeypatch, fresh_engine): """Test truncate tables reports blocking foreign key references.""" - engine = sa.create_engine(f"sqlite:///{tmp_path / 'truncate_fk.db'}", future=True) + engine = fresh_engine create_missing_tables(engine, vocabulary_included=True) monkeypatch.setattr(truncate_tables_module, "require_backend_support", lambda *args, **kwargs: None) diff --git a/tests/test_vocab_results_role_wiring_postgres.py b/tests/test_vocab_results_role_wiring_postgres.py new file mode 100644 index 0000000..6ef7e46 --- /dev/null +++ b/tests/test_vocab_results_role_wiring_postgres.py @@ -0,0 +1,184 @@ +"""Vocab/results-role wiring, exercised on OMOP_Alchemy's own models (Phase 3.2). + +Phase 3's vocab/results-role fix tags every vocabulary table with +``schema=Role.VOCAB.value`` and every derived/results table with +``schema=Role.RESULTS.value``. This is its acceptance test: a single +Postgres connection configured with three genuinely different schema +names for ``schema_name``/``vocab_schema``/``results_schema``, confirming +``create_all`` (now role-aware) places each table in the schema its role +says it belongs in, and that a join across a clinical table's concept_id +FK into the vocab schema compiles and executes correctly in one query -- +the same-connection case, where this is a single eager join, unlike the +split-connection case covered separately in omop-graph's +``test_vocab_split_connection.py``. +""" + +from __future__ import annotations + +from contextlib import ExitStack +from datetime import date +from typing import Iterator, NamedTuple + +import pytest +import sqlalchemy as sa +import sqlalchemy.orm as so + +from oa_configurator.testing import isolated_test_schema + +from omop_alchemy.cdm.model.clinical import Observation, Person +from omop_alchemy.cdm.model.derived import Cohort +from omop_alchemy.cdm.model.vocabulary import Concept, Concept_Class, Domain, Vocabulary +from omop_alchemy.maintenance.cli_schema_tables import create_missing_tables + +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + +_TODAY = date(2020, 1, 1) +META_CONCEPT_ID = 0 + + +class _ThreeSchema(NamedTuple): + engine: sa.Engine + clinical_schema: str + vocab_schema: str + results_schema: str + + +@pytest.fixture() +def three_schema(pg_engine: sa.Engine) -> Iterator[_ThreeSchema]: + with ExitStack() as stack: + clinical_schema = stack.enter_context(isolated_test_schema(pg_engine, prefix="phase32_clinical")) + vocab_schema = stack.enter_context(isolated_test_schema(pg_engine, prefix="phase32_vocab")) + results_schema = stack.enter_context(isolated_test_schema(pg_engine, prefix="phase32_results")) + + engine = pg_engine.execution_options( + schema_translate_map={ + None: clinical_schema, + "vocab": vocab_schema, + "results": results_schema, + } + ) + yield _ThreeSchema( + engine=engine, + clinical_schema=clinical_schema, + vocab_schema=vocab_schema, + results_schema=results_schema, + ) + + +def _bootstrap_vocab(engine: sa.Engine, vocab_schema: str) -> None: + """Insert the minimal, real-FK-checked concept-0 bootstrap that every + concept_id-defaulting column (Person.gender_concept_id and friends, + Observation's required concept FKs) depends on. Domain/Vocabulary/ + Concept_Class/Concept form a genuine insert cycle in Postgres, and + disabling triggers for the load and re-enabling them afterwards is the + same technique production bulk-loads use for this exact reason.""" + vocab_tables = ("domain", "vocabulary", "concept_class", "concept") + with engine.begin() as conn: + for table in vocab_tables: + conn.execute(sa.text(f'ALTER TABLE "{vocab_schema}"."{table}" DISABLE TRIGGER ALL')) + + with so.Session(engine) as session: + session.add_all( + [ + Concept( + concept_id=META_CONCEPT_ID, + concept_name="Meta concept", + domain_id="Metadata", + vocabulary_id="OMOP", + concept_class_id="Metadata", + standard_concept="S", + concept_code="META", + valid_start_date=_TODAY, + valid_end_date=date(2099, 12, 31), + ), + Domain(domain_id="Metadata", domain_name="Metadata", domain_concept_id=META_CONCEPT_ID), + Vocabulary( + vocabulary_id="OMOP", + vocabulary_name="OMOP", + vocabulary_reference="local", + vocabulary_version="test", + vocabulary_concept_id=META_CONCEPT_ID, + ), + Concept_Class( + concept_class_id="Metadata", + concept_class_name="Metadata", + concept_class_concept_id=META_CONCEPT_ID, + ), + ] + ) + session.commit() + + with engine.begin() as conn: + for table in vocab_tables: + conn.execute(sa.text(f'ALTER TABLE "{vocab_schema}"."{table}" ENABLE TRIGGER ALL')) + + +def test_tables_land_in_the_schema_their_role_declares(three_schema: _ThreeSchema) -> None: + create_missing_tables( + three_schema.engine, db_schema=three_schema.clinical_schema, vocabulary_included=True + ) + + inspector = sa.inspect(three_schema.engine) + assert inspector.has_table("person", schema=three_schema.clinical_schema) + assert inspector.has_table("observation", schema=three_schema.clinical_schema) + assert inspector.has_table("concept", schema=three_schema.vocab_schema) + assert inspector.has_table("domain", schema=three_schema.vocab_schema) + assert inspector.has_table("cohort", schema=three_schema.results_schema) + assert inspector.has_table("observation_period", schema=three_schema.results_schema) + + # And not duplicated into the wrong schema. + assert not inspector.has_table("concept", schema=three_schema.clinical_schema) + assert not inspector.has_table("cohort", schema=three_schema.clinical_schema) + + +def test_clinical_to_vocab_join_compiles_and_executes_in_one_query( + three_schema: _ThreeSchema, +) -> None: + create_missing_tables( + three_schema.engine, db_schema=three_schema.clinical_schema, vocabulary_included=True + ) + _bootstrap_vocab(three_schema.engine, three_schema.vocab_schema) + + with so.Session(three_schema.engine) as session: + session.add( + Person( + person_id=1, + year_of_birth=1990, + gender_concept_id=META_CONCEPT_ID, + race_concept_id=META_CONCEPT_ID, + ethnicity_concept_id=META_CONCEPT_ID, + ) + ) + session.commit() + + session.add( + Observation( + observation_id=1, + person_id=1, + observation_concept_id=META_CONCEPT_ID, + observation_type_concept_id=META_CONCEPT_ID, + observation_date=_TODAY, + ) + ) + session.add( + Cohort( + cohort_definition_id=1, + subject_id=1, + cohort_start_date=_TODAY, + cohort_end_date=_TODAY, + ) + ) + session.commit() + + row = session.execute( + sa.select(Observation.observation_id, Concept.concept_name).join( + Concept, Observation.observation_concept_id == Concept.concept_id + ) + ).one() + assert row.observation_id == 1 + assert row.concept_name == "Meta concept" + + cohort_row = session.execute( + sa.select(Cohort.cohort_definition_id).where(Cohort.subject_id == 1) + ).one() + assert cohort_row.cohort_definition_id == 1 From b57f976027df6463dad243617b21c2b6c7b5629b Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 1 Sep 2026 06:11:38 +0000 Subject: [PATCH 02/32] Update CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b307965..0b40f63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,4 +40,4 @@ jobs: --connection pg_test \ --schema-name public uv run omop-config configure omop_alchemy \ - --test-cdm-db test_cdm_db + --test-cdm-db-pg test_cdm_db From adbf623580f26a7595c7c756b09ac37c5074878b Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 3 Sep 2026 04:05:05 +0000 Subject: [PATCH 03/32] Drop docker --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index d20e252..0000000 --- a/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM python:3.12-slim -RUN pip install --no-cache-dir ".[postgres]" -WORKDIR /workspace From 40713116cd7ee7e005ffd446e4c62520261a8cd3 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 3 Sep 2026 05:44:47 +0000 Subject: [PATCH 04/32] Update CI to new cava-devops --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b40f63..2c7e5f3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,7 +22,7 @@ jobs: with: ty-src: omop_alchemy build-test-postgres: - uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-postgres.yml@main + uses: AustralianCancerDataNetwork/cava-devops/.github/workflows/build-test-postgres-v2.yml@main with: ty-src: omop_alchemy postgres-db: test_db From cfe541b0cdf96840d2893ddeccc06d5bda56500f Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 3 Sep 2026 05:58:04 +0000 Subject: [PATCH 05/32] Proper schema qualification, adapted CLI to user proper resolved values, delegate some functionality to the backends, add the option to reconcile a schema and rectify it if there are stray tables --- omop_alchemy/backends/base.py | 15 ++ omop_alchemy/backends/postgres.py | 15 ++ omop_alchemy/maintenance/_cli_utils.py | 42 ++--- omop_alchemy/maintenance/cli_backup.py | 4 +- omop_alchemy/maintenance/cli_foreign_keys.py | 8 +- omop_alchemy/maintenance/cli_fulltext.py | 6 +- omop_alchemy/maintenance/cli_indexes.py | 10 +- omop_alchemy/maintenance/cli_schema.py | 115 ++++++++++++-- omop_alchemy/maintenance/cli_schema_doctor.py | 18 +++ .../maintenance/cli_schema_reconcile.py | 124 ++++++++++++--- .../maintenance/cli_schema_rectify.py | 98 ++++++++++++ omop_alchemy/maintenance/cli_schema_tables.py | 28 +++- omop_alchemy/maintenance/cli_tables.py | 6 +- omop_alchemy/maintenance/cli_vocab.py | 99 ++++++------ tests/test_schema_provenance_guard.py | 115 ++++++++++++++ tests/test_schema_reconcile.py | 135 +++++++++++++--- tests/test_schema_rectify_cli.py | 149 ++++++++++++++++++ 17 files changed, 837 insertions(+), 150 deletions(-) create mode 100644 omop_alchemy/maintenance/cli_schema_rectify.py create mode 100644 tests/test_schema_provenance_guard.py create mode 100644 tests/test_schema_rectify_cli.py diff --git a/omop_alchemy/backends/base.py b/omop_alchemy/backends/base.py index 634ba59..f5a72ab 100644 --- a/omop_alchemy/backends/base.py +++ b/omop_alchemy/backends/base.py @@ -123,6 +123,21 @@ def get_clustered_index_name( ) -> str | None: raise FeatureNotSupportedError("Cluster index inspection", self) + # ── Row counts ─────────────────────────────────────────────────────────── + + def approximate_row_counts( + self, + conn: sa.Connection, + schema: str, + ) -> dict[str, int]: + """Cheap, catalog-based row-count estimate per table in schema. + + Unlike most Backend methods, schema is explicit rather than read via + schema_of(conn): a caller previewing an orphan schema is inspecting a + schema other than the connection's own configured one. + """ + raise FeatureNotSupportedError("Approximate row counts", self) + # ── Table operations ───────────────────────────────────────────────────── @abstractmethod diff --git a/omop_alchemy/backends/postgres.py b/omop_alchemy/backends/postgres.py index f5bc86d..c1d6d0d 100644 --- a/omop_alchemy/backends/postgres.py +++ b/omop_alchemy/backends/postgres.py @@ -126,6 +126,21 @@ def get_clustered_index_name( ).scalar_one_or_none() return str(result) if result is not None else None + # ── Row counts ─────────────────────────────────────────────────────────── + + def approximate_row_counts( + self, + conn: sa.Connection, + schema: str, + ) -> dict[str, int]: + rows = conn.execute( + sa.text( + "SELECT relname, n_live_tup FROM pg_stat_user_tables WHERE schemaname = :schema" + ), + {"schema": schema}, + ).all() + return {row.relname: row.n_live_tup for row in rows} + # ── Table operations ───────────────────────────────────────────────────── def analyze_table( diff --git a/omop_alchemy/maintenance/_cli_utils.py b/omop_alchemy/maintenance/_cli_utils.py index e7e2dba..27c9b41 100644 --- a/omop_alchemy/maintenance/_cli_utils.py +++ b/omop_alchemy/maintenance/_cli_utils.py @@ -8,9 +8,8 @@ from enum import StrEnum from typing import Any, Callable, TypeVar -import sqlalchemy as sa import typer -from oa_configurator import register_reserved_schema +from oa_configurator import ResolvedCDMDatabase, register_reserved_schema from sqlalchemy.exc import SQLAlchemyError from .tables import TableCategory @@ -74,14 +73,12 @@ def __new__(cls, code: str, severity: Severity): def __init__(self, code: str, severity: Severity): self.severity = severity - # -- shared across every dry-run/apply-style domain -- + # shared across every dry-run/apply-style domain PLANNED = ("planned", Severity.INFO) APPLIED = ("applied", Severity.OK) SKIPPED = ("skipped", Severity.WARNING) - # -- domain-specific "applied" words (backup, index restore/capture, - # sequence reset, vocab load) -- same OK severity as APPLIED, kept as - # distinct words since the specific outcome is worth seeing at a glance -- + # domain-specific "applied" words CREATED = ("created", Severity.OK) LOADED = ("loaded", Severity.OK) RESET = ("reset", Severity.OK) @@ -91,15 +88,16 @@ def __init__(self, code: str, severity: Severity): PASSED = ("passed", Severity.OK) MATCHED = ("matched", Severity.OK) - # -- warnings: something worth a look, but not blocking -- + # warnings: something worth a look, but not blocking WARNING = ("warning", Severity.WARNING) LIMITED = ("limited", Severity.WARNING) DRIFTED = ("drifted", Severity.WARNING) - # -- informational: an intentionally supported state -- + # informational: an intentionally supported state RENAMED = ("renamed", Severity.INFO) - # -- errors/failures -- + # errors/failures + RELOCATED = ("relocated", Severity.ERROR) MISSING = ("missing", Severity.ERROR) UNEXPECTED = ("unexpected", Severity.ERROR) MISMATCH = ("mismatch", Severity.ERROR) @@ -110,18 +108,16 @@ def __init__(self, code: str, severity: Severity): @dataclass(frozen=True) class _ConnContext: - """Connection context derived from the oa_configurator resolved resource. + """Connection context assembled once per CLI command. - ``vocab_engine`` is the same object as ``engine`` unless ``vocab_connection`` - names a physically different server; only DDL creating vocab-role tables - needs it instead of the primary engine. + resource_name and athena_source come from OmopAlchemyConfig, not from + resolved. Everything else a command needs (schema, vocab/results + schema, test_only, a vocab engine) is available via resolved directly, + or via resolved.vocab_engine_for(engine), rather than duplicated here. """ - db_schema: str | None - engine_url: str = "" + resolved: ResolvedCDMDatabase resource_name: str = "" - athena_source: str | None = None # from OmopAlchemyConfig.athena_source_path - vocab_engine: sa.Engine | None = None - vocab_schema: str | None = None + athena_source: str | None = None # ── Decorator ───────────────────────────────────────────────────────────────── @@ -151,20 +147,16 @@ def wrapper(**kwargs: Any) -> Any: from ..config import create_cdm_engine, get_cdm_context pkg_config, resolved = get_cdm_context() engine = create_cdm_engine(resolved) - vocab_engine = resolved.vocab_engine_for(engine) conn = _ConnContext( - db_schema=resolved.schema_name, - engine_url=engine.url.render_as_string(hide_password=True), + resolved=resolved, resource_name=pkg_config.cdm_db, - vocab_engine=vocab_engine, - vocab_schema=resolved.vocab_schema, athena_source=pkg_config.athena_source_path, ) console.print( render_command_header( command_name=command_name, - engine_url=conn.engine_url, - db_schema=conn.db_schema, + engine_url=engine.url.render_as_string(hide_password=True), + db_schema=resolved.schema_name, vocabulary_included=_vocab, mode_label=_mode, ) diff --git a/omop_alchemy/maintenance/cli_backup.py b/omop_alchemy/maintenance/cli_backup.py index d9acc22..fc9b3f8 100644 --- a/omop_alchemy/maintenance/cli_backup.py +++ b/omop_alchemy/maintenance/cli_backup.py @@ -175,7 +175,7 @@ def backup_database_command( engine, output_path=output_path, backup_format=backup_format, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, dry_run=dry_run, ) console.print(render_backup_result(result)) @@ -200,7 +200,7 @@ def restore_database_command( engine, input_path=input_path, backup_format=backup_format, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, dry_run=dry_run, ) console.print(render_restore_result(result)) diff --git a/omop_alchemy/maintenance/cli_foreign_keys.py b/omop_alchemy/maintenance/cli_foreign_keys.py index 1d69cd3..2e3ce91 100644 --- a/omop_alchemy/maintenance/cli_foreign_keys.py +++ b/omop_alchemy/maintenance/cli_foreign_keys.py @@ -415,7 +415,7 @@ def disable_foreign_keys_command( results = manage_foreign_key_triggers( engine, enable=False, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, strict=strict, @@ -452,7 +452,7 @@ def enable_foreign_keys_command( results = manage_foreign_key_triggers( engine, enable=True, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, strict=strict, @@ -477,7 +477,7 @@ def foreign_key_status_command( with console.status("Inspecting foreign key trigger status..."): results = collect_foreign_key_trigger_status( engine, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, ) console.print(render_foreign_key_status_results(results)) @@ -499,7 +499,7 @@ def foreign_key_validate_command( with console.status("Validating selected foreign key relationships..."): report = validate_foreign_key_constraints( engine, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, ) console.print(render_foreign_key_validation_results(report.results)) diff --git a/omop_alchemy/maintenance/cli_fulltext.py b/omop_alchemy/maintenance/cli_fulltext.py index c4793a4..b3c9046 100644 --- a/omop_alchemy/maintenance/cli_fulltext.py +++ b/omop_alchemy/maintenance/cli_fulltext.py @@ -221,7 +221,7 @@ def install_fulltext_command( with console.status("Managing PostgreSQL full-text sidecar columns..."): results = install_fulltext_columns( engine, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, create_indexes=create_indexes, fastupdate=fastupdate, dry_run=dry_run, @@ -245,7 +245,7 @@ def populate_fulltext_command( with console.status("Managing PostgreSQL full-text sidecar columns..."): results = populate_fulltext_columns( engine, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, regconfig=regconfig, dry_run=dry_run, ) @@ -269,7 +269,7 @@ def drop_fulltext_command( with console.status("Managing PostgreSQL full-text sidecar columns..."): results = drop_fulltext_columns( engine, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, drop_indexes=drop_indexes, dry_run=dry_run, ) diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index 841dba0..4aece88 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -957,7 +957,7 @@ def disable_indexes_command( results = manage_indexes( engine, enable=False, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, ) @@ -993,7 +993,7 @@ def enable_indexes_command( results = manage_indexes( engine, enable=True, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, cluster=cluster, @@ -1018,7 +1018,7 @@ def cluster_tables_command( """CLUSTER tables using their ORM-designated cluster index. Physically rewrites table data sorted by the cluster index for improved sequential-scan - performance. Requires approximately 2× the table size in free disk space per table. + performance. Requires approximately 2x the table size in free disk space per table. Run this after 'indexes enable' once you have confirmed sufficient disk headroom. On Docker, check Docker Desktop → Resources → Virtual Disk Limit before running on @@ -1034,7 +1034,7 @@ def cluster_tables_command( results: list[IndexManagementResult] = [] for table in selected_tables: - if not inspector.has_table(table.table_name, schema=conn.db_schema): + if not inspector.has_table(table.table_name, schema=conn.resolved.schema_name): continue cluster_index_name = _cluster_target_name(table) @@ -1042,7 +1042,7 @@ def cluster_tables_command( continue cluster_columns = _cluster_column_names(table, cluster_index_name) - existing_indexes = inspector.get_indexes(table.table_name, schema=conn.db_schema) + existing_indexes = inspector.get_indexes(table.table_name, schema=conn.resolved.schema_name) physical_cluster_name = _resolve_physical_cluster_name( existing_indexes, cluster_index_name, diff --git a/omop_alchemy/maintenance/cli_schema.py b/omop_alchemy/maintenance/cli_schema.py index 9ab476a..2f33c1c 100644 --- a/omop_alchemy/maintenance/cli_schema.py +++ b/omop_alchemy/maintenance/cli_schema.py @@ -1,10 +1,11 @@ -"""Schema subapp: thin shim re-exporting all domain types and wiring five CLI commands.""" +"""Schema subapp: thin shim re-exporting all domain types and wiring seven CLI commands.""" from __future__ import annotations import typer +from oa_configurator import Resolver, Role, load_stack_config, record_schema_provenance -from ._cli_utils import omop_command +from ._cli_utils import handle_error, omop_command from .cli_schema_doctor import ( DoctorCheck as DoctorCheck, DoctorReport as DoctorReport, @@ -23,6 +24,10 @@ TableReconciliationResult as TableReconciliationResult, reconcile_schema, ) +from .cli_schema_rectify import ( + OrphanTablePreview as OrphanTablePreview, + drop_orphan_schema_tables, +) from .cli_schema_summary import ( TableSummaryResult as TableSummaryResult, collect_data_summary, @@ -101,7 +106,8 @@ def doctor_command( with console.status("Running maintenance doctor checks..."): report = collect_doctor_report( engine=engine, - db_schema=conn.db_schema, + resolved=conn.resolved, + db_schema=conn.resolved.schema_name, resource_name=conn.resource_name, vocabulary_included=vocabulary_included, deep=deep, @@ -128,7 +134,7 @@ def reconcile_schema_command( ) -> None: """Compare ORM-managed SQLAlchemy metadata against the current target database schema.""" with console.status("Reconciling ORM metadata against target database schema..."): - report = reconcile_schema(engine, db_schema=conn.db_schema, vocabulary_included=vocabulary_included) + report = reconcile_schema(engine, resolved=conn.resolved, vocabulary_included=vocabulary_included) console.print(render_reconciliation_results(report.table_results)) console.print(render_reconciliation_issues(report.issues)) console.print(render_reconciliation_summary(report)) @@ -147,14 +153,21 @@ def create_missing_tables_command( dry_run: bool = False, ) -> None: """Create missing ORM-managed OMOP tables from metadata.""" - with console.status("Creating missing tables..."): - results = create_missing_tables( - engine, - vocab_engine=conn.vocab_engine, - db_schema=conn.db_schema, - vocabulary_included=vocabulary_included, - dry_run=dry_run, - ) + vocab_engine = conn.resolved.vocab_engine_for(engine) + try: + with console.status("Creating missing tables..."): + results = create_missing_tables( + engine, + vocab_engine=vocab_engine, + db_schema=conn.resolved.schema_name, + vocabulary_included=vocabulary_included, + dry_run=dry_run, + resolved=conn.resolved, + test_only=conn.resolved.connection.test_only, + ) + finally: + if vocab_engine is not engine: + vocab_engine.dispose() console.print(render_table_creation_results(results)) console.print(render_table_creation_summary(results, dry_run=dry_run)) @@ -179,9 +192,85 @@ def data_summary_command( with console.status("Collecting table summary..."): results = collect_data_summary( engine, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, existing_only=not include_missing, ) console.print(render_data_summary_results(results)) console.print(render_data_summary_summary(results)) + + +@app.command("acknowledge-schema-migration") +def acknowledge_schema_migration_command( + database: str = typer.Option(..., "--database", help="Name of the [databases.*] entry to acknowledge."), + role: Role = typer.Option(Role.PRIMARY, "--role", help="Logical role whose schema is being acknowledged."), + new_schema: str = typer.Option(..., "--new-schema", help="Schema to record as the accepted baseline."), + reason: str = typer.Option( + ..., + "--reason", + help="Free-text justification for this acknowledgment. Mandatory: there is no --yes shortcut.", + ), +) -> None: + """Record a schema as the deliberate baseline for a database/role. + + Overwrites any existing provenance row (its prior value moves to + previous_schema); does not touch the CDM tables themselves. + """ + try: + stack = load_stack_config() + resolved = Resolver(stack).resolve_database(database) + engine = resolved.create_engine(role=role) + try: + with engine.begin() as connection: + record_schema_provenance( + connection, resolved, role=role, new_schema=new_schema, reason=reason + ) + finally: + engine.dispose() + except Exception as exc: + handle_error(exc) + console.print( + f"[green]Acknowledged[/green] {database!r} (role {role.value!r}) -> schema {new_schema!r}." + ) + + +@app.command("drop-orphan-schema-tables") +def drop_orphan_schema_tables_command( + database: str = typer.Option(..., "--database", help="Name of the [databases.*] entry providing the connection."), + schema: str = typer.Option(..., "--schema", help="Orphan schema to inspect/drop tables from."), + role: Role = typer.Option(Role.PRIMARY, "--role", help="Logical role providing the connection to use."), + confirm: bool = typer.Option( + False, + "--confirm", + help="Actually drop the previewed tables. Omit to preview only.", + ), +) -> None: + """Drop tables physically found in an orphaned schema, after a stack-wide safety check. + + Refuses if the named schema is still the current schema target of any + configured database/role, not just the one named here. Without + --confirm, only previews what would be dropped. + """ + try: + stack = load_stack_config() + resolved = Resolver(stack).resolve_database(database) + engine = resolved.create_engine(role=role) + try: + with engine.begin() as connection: + preview = drop_orphan_schema_tables( + connection, stack=stack, orphan_schema=schema, confirm=confirm + ) + finally: + engine.dispose() + + if not preview: + console.print(f"No tables found in schema {schema!r}.") + return + for item in preview: + count = "unknown" if item.approximate_row_count is None else str(item.approximate_row_count) + verb = "Dropped" if confirm else "Would drop" + console.print(f"{verb} {schema}.{item.table_name} (~{count} rows)") + if not confirm: + console.print("[yellow]Preview only. Re-run with --confirm to actually drop these tables.[/yellow]") + except Exception as exc: + handle_error(exc) diff --git a/omop_alchemy/maintenance/cli_schema_doctor.py b/omop_alchemy/maintenance/cli_schema_doctor.py index 6fe5ad3..010dcd8 100644 --- a/omop_alchemy/maintenance/cli_schema_doctor.py +++ b/omop_alchemy/maintenance/cli_schema_doctor.py @@ -5,6 +5,7 @@ from dataclasses import dataclass import sqlalchemy as sa +from oa_configurator import ResolvedDatabase from omop_alchemy.backends.resolve import SupportedDialect @@ -101,6 +102,18 @@ def _build_recommendations( action="Review `omop-alchemy reconcile-schema` output before continuing with ETL or maintenance work.", ) ) + if any(issue.status == Status.RELOCATED for issue in reconciliation.issues): + recommendations.append( + DoctorRecommendation( + status=Status.WARNING, + summary="Some tables were found under a different schema than expected.", + action=( + "Run `omop-alchemy acknowledge-schema-migration` if this was a " + "deliberate change, or `omop-alchemy drop-orphan-schema-tables` to " + "clean up an orphaned copy." + ), + ) + ) if foreign_key_status is not None and any( item.disabled_trigger_count > 0 for item in foreign_key_status @@ -165,6 +178,7 @@ def _build_recommendations( def collect_doctor_report( *, engine: sa.engine.Engine, + resolved: ResolvedDatabase | None = None, db_schema: str | None = None, resource_name: str | None = None, vocabulary_included: bool = True, @@ -178,6 +192,9 @@ def collect_doctor_report( Already-resolved CDM engine (e.g. from the ``@omop_command`` decorator), reused for all database checks instead of re-resolving config. The caller retains ownership; this function does not dispose it. + resolved : ResolvedDatabase, optional + Forwarded to reconcile_schema (--deep only) so vocab/results tables + are compared against their own schema, not db_schema uniformly. db_schema : str, optional CDM schema associated with ``engine``. Omit to use its default schema. resource_name : str, optional @@ -225,6 +242,7 @@ def collect_doctor_report( if deep: reconciliation = reconcile_schema( engine, + resolved=resolved, db_schema=db_schema, vocabulary_included=vocabulary_included, ) diff --git a/omop_alchemy/maintenance/cli_schema_reconcile.py b/omop_alchemy/maintenance/cli_schema_reconcile.py index 57d2f01..03ac9a7 100644 --- a/omop_alchemy/maintenance/cli_schema_reconcile.py +++ b/omop_alchemy/maintenance/cli_schema_reconcile.py @@ -5,7 +5,7 @@ from dataclasses import dataclass import sqlalchemy as sa -from oa_configurator import supports_schemas +from oa_configurator import ResolvedDatabase, Role, find_table_in_other_schemas, supports_schemas from sqlalchemy.engine.interfaces import ReflectedForeignKeyConstraint, ReflectedIndex from ..backends import backend_supports, resolve_backend @@ -67,19 +67,65 @@ class SchemaReconciliationReport: issues: tuple[ReconciliationIssue, ...] -def _schema_table(table: sa.Table, db_schema: str | None) -> sa.Table: - """Return table unchanged when db_schema is None, or a schema-qualified copy when a schema is specified.""" - if db_schema is None: - return table +def _role_from_schema_tag(schema_tag: str | None) -> Role: + """Map a table's declared .schema tag (None/"vocab"/"results") to its Role.""" + if schema_tag == Role.VOCAB.value: + return Role.VOCAB + if schema_tag == Role.RESULTS.value: + return Role.RESULTS + return Role.PRIMARY + +def _effective_schema( + resolved: ResolvedDatabase | None, role: Role, db_schema: str | None +) -> str | None: + """resolved.schema_for_role(role) when resolved is given, else db_schema + applied the same regardless of role. The fallback for a caller with no + resolved object to hand (e.g. a test built directly against a bare engine). + """ + return resolved.schema_for_role(role) if resolved is not None else db_schema + + +def _schema_qualified_tables( + resolved: ResolvedDatabase | None, db_schema: str | None +) -> dict[int, sa.Table]: + """Schema-qualified copy of every table in Base.metadata, keyed by id() of the original. + + Notes + ----- + Each table is qualified to its own role's schema via resolved, not one + blanket value: a vocab-role table can live in a different physical schema + than a clinical one. + + Copied together into one MetaData() in a single pass: to_metadata() never + brings a referenced table along on its own, and resolving an FK's target + needs that table's copy already present in the same metadata. The whole + Base.metadata is copied, not just the selected/diffed subset, since a + selected table can reference one excluded from the diff itself (e.g. a + vocabulary FK target when vocabulary_included=False). + + Returns the tables unchanged (keyed by their own id) when both resolved + and db_schema are None. + """ + from orm_loader.helpers import Base + + if resolved is None and db_schema is None: + return {id(table): table for table in Base.metadata.tables.values()} metadata = sa.MetaData() - return table.to_metadata( - metadata, - schema=db_schema, - referred_schema_fn=( - lambda _table, to_schema, _constraint, _referred_schema: to_schema - ), - ) + return { + id(table): table.to_metadata( + metadata, + # SQLAlchemy's own stub omits None from schema's declared type, + # despite accepting and correctly handling it at runtime + schema=_effective_schema(resolved, _role_from_schema_tag(table.schema), db_schema), # ty: ignore[invalid-argument-type] + referred_schema_fn=( + lambda _table, _to_schema, _constraint, referred_schema: _effective_schema( + resolved, _role_from_schema_tag(referred_schema), db_schema + ) + ), + ) + for table in Base.metadata.tables.values() + } def _normalized_type(type_: sa.types.TypeEngine[object], dialect: sa.engine.Dialect) -> str: @@ -140,16 +186,23 @@ def _actual_indexes( def reconcile_schema( engine: sa.Engine, *, + resolved: ResolvedDatabase | None = None, db_schema: str | None = None, vocabulary_included: bool = False, ) -> SchemaReconciliationReport: - """Compare ORM metadata against the live database schema. Reports missing columns, indexes, FKs, and cluster state.""" + """Compare ORM metadata against the live database schema. Reports missing columns, indexes, FKs, and cluster state. + + resolved, when given, qualifies each table to its own role's schema + (schema_name/vocab_schema/results_schema) rather than applying db_schema + to every table regardless of role. + """ excluded_categories: tuple[TableCategory, ...] = ( () if vocabulary_included else (TableCategory.VOCABULARY,) ) _backend = resolve_backend(engine) _cross_schema_fk_supported = supports_schemas(engine) selected_tables = select_maintenance_tables(exclude_categories=excluded_categories) + schema_qualified_tables = _schema_qualified_tables(resolved, db_schema) inspector = sa.inspect(engine) all_issues: list[ReconciliationIssue] = [] table_results: list[TableReconciliationResult] = [] @@ -157,8 +210,41 @@ def reconcile_schema( with engine.connect() as connection: for maintenance_table in selected_tables: table_issues: list[ReconciliationIssue] = [] - exists = inspector.has_table(maintenance_table.table_name, schema=db_schema) + table_role = _role_from_schema_tag(maintenance_table.table.schema) + table_schema = _effective_schema(resolved, table_role, db_schema) + exists = inspector.has_table(maintenance_table.table_name, schema=table_schema) if not exists: + relocated_to = find_table_in_other_schemas( + engine, maintenance_table.table_name, expected_schema=table_schema + ) + if relocated_to: + detail = ( + f"Table is absent from schema {table_schema!r} but found in " + f"{', '.join(sorted(relocated_to))!r}." + ) + table_issues.append( + ReconciliationIssue( + table_name=maintenance_table.table_name, + category=maintenance_table.category, + component="table", + object_name=maintenance_table.table_name, + status=Status.RELOCATED, + expected=table_schema, + actual=", ".join(sorted(relocated_to)), + detail=detail, + ) + ) + table_results.append( + TableReconciliationResult( + table_name=maintenance_table.table_name, + category=maintenance_table.category, + status=Status.RELOCATED, + issue_count=1, + detail=detail, + ) + ) + all_issues.extend(table_issues) + continue table_issues.append( ReconciliationIssue( table_name=maintenance_table.table_name, @@ -183,14 +269,14 @@ def reconcile_schema( all_issues.extend(table_issues) continue - expected_table = _schema_table(maintenance_table.table, db_schema) + expected_table = schema_qualified_tables[id(maintenance_table.table)] expected_columns = {column.name: column for column in expected_table.columns} actual_columns = { str(column["name"]): column - for column in inspector.get_columns(maintenance_table.table_name, schema=db_schema) + for column in inspector.get_columns(maintenance_table.table_name, schema=table_schema) } actual_pk_names = tuple( - inspector.get_pk_constraint(maintenance_table.table_name, schema=db_schema).get("constrained_columns") or [] + inspector.get_pk_constraint(maintenance_table.table_name, schema=table_schema).get("constrained_columns") or [] ) expected_pk_names = tuple(column.name for column in expected_table.primary_key.columns) @@ -274,7 +360,7 @@ def reconcile_schema( ) expected_fks = _expected_foreign_keys(expected_table) - actual_fks = _actual_foreign_keys(inspector, maintenance_table.table_name, db_schema) + actual_fks = _actual_foreign_keys(inspector, maintenance_table.table_name, table_schema) for signature, constraint in expected_fks.items(): if signature not in actual_fks: @@ -315,7 +401,7 @@ def reconcile_schema( ) expected_idxs = _expected_indexes(expected_table) - actual_idxs = _actual_indexes(inspector, maintenance_table.table_name, db_schema) + actual_idxs = _actual_indexes(inspector, maintenance_table.table_name, table_schema) actual_index_list = list(actual_idxs.values()) renamed_actual_names: set[str] = set() diff --git a/omop_alchemy/maintenance/cli_schema_rectify.py b/omop_alchemy/maintenance/cli_schema_rectify.py new file mode 100644 index 0000000..5f32bbf --- /dev/null +++ b/omop_alchemy/maintenance/cli_schema_rectify.py @@ -0,0 +1,98 @@ +"""Rectify domain: acknowledge a deliberate schema change, or clean up orphaned +tables left behind by one. + +Every action requires an explicit target and confirmation; nothing here +auto-detects or auto-deletes. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import sqlalchemy as sa +from oa_configurator import ResolvedCDMDatabase, Resolver, StackConfig, schema_inspect + +from ..backends import backend_supports, resolve_backend + + +@dataclass(frozen=True) +class OrphanTablePreview: + """One table found in a candidate orphan schema, with an approximate row count.""" + + table_name: str + approximate_row_count: int | None + + +def preview_orphan_schema_tables(connection: sa.Connection, schema: str) -> list[OrphanTablePreview]: + """List tables physically present in schema, with an approximate row count + where the backend supports one (None otherwise). + """ + table_names = schema_inspect(connection, schema=schema).get_table_names() + backend = resolve_backend(connection.engine) + counts: dict[str, int] = ( + backend.approximate_row_counts(connection, schema) + if backend_supports(backend, "approximate_row_counts") + else {} + ) + return [ + OrphanTablePreview(table_name=name, approximate_row_count=counts.get(name)) + for name in table_names + ] + + +def schema_is_a_current_target(stack: StackConfig, schema: str) -> str | None: + """Return the name of a configured database whose current schema (or + vocab/results schema) equals schema, or None. + + Checks every database in the stack, not just the one named on the + command line, since two CDM databases can legitimately share one + vocabulary schema; dropping tables there would destroy a database a + different entry is actively using. + """ + resolver = Resolver(stack) + for name in stack.databases: + try: + resolved = resolver.resolve_database(name) + except Exception: + continue + candidates: set[str | None] = {resolved.schema_name} + if isinstance(resolved, ResolvedCDMDatabase): + candidates.add(resolved.vocab_schema) + candidates.add(resolved.results_schema) + if schema in candidates: + return name + return None + + +def drop_orphan_schema_tables( + connection: sa.Connection, + *, + stack: StackConfig, + orphan_schema: str, + confirm: bool, +) -> list[OrphanTablePreview]: + """Drop every table found in *orphan_schema*, after the cross-entry safety check. + + Preview-only when *confirm* is False: returns what would be dropped + without touching anything. + + Raises + ------ + RuntimeError + If *orphan_schema* is the current schema target of any configured + database/role in *stack*. + """ + blocking = schema_is_a_current_target(stack, orphan_schema) + if blocking is not None: + raise RuntimeError( + f"Refusing to drop tables in schema {orphan_schema!r}: it is the current schema " + f"target of database {blocking!r}. Reconfigure or drop that database entry first " + "if this schema is genuinely meant to be retired." + ) + preview = preview_orphan_schema_tables(connection, orphan_schema) + if confirm: + for item in preview: + connection.execute( + sa.text(f'DROP TABLE IF EXISTS "{orphan_schema}"."{item.table_name}" CASCADE') + ) + return preview diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index a6a5227..44316b3 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from oa_configurator import Role, ensure_schema +from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance from orm_loader.helpers import Base from ._cli_utils import Status, dry_label, dry_status from .tables import ( @@ -61,6 +61,8 @@ def create_missing_tables( db_schema: str | None = None, vocabulary_included: bool = True, dry_run: bool = False, + resolved: ResolvedCDMDatabase | None = None, + test_only: bool = False, ) -> list[TableCreationResult]: """Create any ORM-managed tables missing from the target database. Skips tables with unresolved FK dependencies. @@ -70,6 +72,13 @@ def create_missing_tables( Engine for vocab-role tables, when ``vocab_connection`` names a physically different server than ``engine``. Defaults to ``engine`` (the common, same-connection case). + resolved : ResolvedCDMDatabase, optional + Enables the schema-provenance guard around each ``create_all()`` + call. Omitted by direct test/programmatic callers that hand in a + bare engine with no resolved config behind it, in which case the + guard no-ops. + test_only : bool, optional + Forwarded to the guard. Ignored when *resolved* is None. """ vocab_engine = vocab_engine if vocab_engine is not None else engine if not dry_run: @@ -105,7 +114,11 @@ def create_missing_tables( all_tables = [table.table for table in creatable_tables] if vocab_engine is engine: # One call: create_all's dependency sort and FK-deferral must see every table together. - with engine.begin() as connection: + with ( + engine.begin() as connection, + guard_schema_provenance(connection, resolved, role=Role.PRIMARY, test_only=test_only), + guard_schema_provenance(connection, resolved, role=Role.RESULTS, test_only=test_only), + ): Base.metadata.create_all( bind=connection, tables=all_tables, checkfirst=True ) @@ -119,12 +132,19 @@ def create_missing_tables( table for table in all_tables if table.schema != Role.VOCAB.value ] if other_tables: - with engine.begin() as connection: + with ( + engine.begin() as connection, + guard_schema_provenance(connection, resolved, role=Role.PRIMARY, test_only=test_only), + guard_schema_provenance(connection, resolved, role=Role.RESULTS, test_only=test_only), + ): Base.metadata.create_all( bind=connection, tables=other_tables, checkfirst=True ) if vocab_tables: - with vocab_engine.begin() as vocab_connection: + with ( + vocab_engine.begin() as vocab_connection, + guard_schema_provenance(vocab_connection, resolved, role=Role.VOCAB, test_only=test_only), + ): Base.metadata.create_all( bind=vocab_connection, tables=vocab_tables, checkfirst=True ) diff --git a/omop_alchemy/maintenance/cli_tables.py b/omop_alchemy/maintenance/cli_tables.py index 0cac324..e04dcf3 100644 --- a/omop_alchemy/maintenance/cli_tables.py +++ b/omop_alchemy/maintenance/cli_tables.py @@ -372,7 +372,7 @@ def analyze_tables_command( with console.status("Refreshing planner statistics for selected tables..."): results = analyze_tables( engine, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, scope=resolved_scope, table_names=resolved_tables, vacuum=vacuum, @@ -402,7 +402,7 @@ def reset_sequences_command( with console.status("Resetting PostgreSQL sequences..."): results = reset_model_sequences( engine, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, ) @@ -461,7 +461,7 @@ def truncate_tables_command( with console.status("Truncating selected tables..."): results = truncate_tables( engine, - db_schema=conn.db_schema, + db_schema=conn.resolved.schema_name, scope=resolved_scope, table_names=resolved_tables, restart_identities=restart_identities, diff --git a/omop_alchemy/maintenance/cli_vocab.py b/omop_alchemy/maintenance/cli_vocab.py index 2167cac..46886fa 100644 --- a/omop_alchemy/maintenance/cli_vocab.py +++ b/omop_alchemy/maintenance/cli_vocab.py @@ -639,7 +639,7 @@ def load_vocab_source_command( staging_chunk_size: int | None = typer.Option( 100_000, help=( - "[Phase 1] Rows per ORM transaction when loading CSV → staging table. " + "Staging load: rows per ORM transaction when loading CSV → staging table. " "Ignored when the PostgreSQL COPY fast-path is active (the default for " "Athena CSVs). Pass 0 to disable chunking entirely." ), @@ -657,7 +657,7 @@ def load_vocab_source_command( merge_batch_size: int | None = typer.Option( None, help=( - "[Phase 2] Rows per transaction when merging staging → target table. " + "Merge: rows per transaction when merging staging → target table. " "Default: None (no pagination — single INSERT per table, fastest for high-RAM systems). " "Set to a positive integer to enable paginated commits for memory-constrained systems; " "note that pagination adds a COUNT query and an index build on the staging table " @@ -678,54 +678,59 @@ def load_vocab_source_command( ) raise typer.Exit(code=1) - with Progress( - SpinnerColumn(), - TextColumn("[bold cyan]{task.description}"), - BarColumn(bar_width=None), - TaskProgressColumn(), - TimeElapsedColumn(), - console=console, - transient=False, - ) as progress: - task_id = progress.add_task( - "Preparing Athena vocabulary load...", total=100.0, completed=0 - ) - completed_tables: list[str] = [] - - def _update_progress(event: VocabularyLoadProgress) -> None: - progress.update( - task_id, completed=event.percent, description=event.description + vocab_engine = conn.resolved.vocab_engine_for(engine) + try: + with Progress( + SpinnerColumn(), + TextColumn("[bold cyan]{task.description}"), + BarColumn(bar_width=None), + TaskProgressColumn(), + TimeElapsedColumn(), + console=console, + transient=False, + ) as progress: + task_id = progress.add_task( + "Preparing Athena vocabulary load...", total=100.0, completed=0 ) - if event.table_done and event.table_name is not None: - completed_tables.append(event.table_name) - row_info = ( - f": [dim]{event.rows_this_table:,} rows[/dim]" - if event.rows_this_table is not None - else "" - ) - progress.console.print( - f"[green]loaded[/green] [bold]{event.table_name}[/bold]{row_info} " - f"({len(completed_tables)}/{event.table_count})" + completed_tables: list[str] = [] + + def _update_progress(event: VocabularyLoadProgress) -> None: + progress.update( + task_id, completed=event.percent, description=event.description ) + if event.table_done and event.table_name is not None: + completed_tables.append(event.table_name) + row_info = ( + f": [dim]{event.rows_this_table:,} rows[/dim]" + if event.rows_this_table is not None + else "" + ) + progress.console.print( + f"[green]loaded[/green] [bold]{event.table_name}[/bold]{row_info} " + f"({len(completed_tables)}/{event.table_count})" + ) - report = load_vocab_source( - engine, - vocab_engine=conn.vocab_engine, - vocab_schema=conn.vocab_schema, - source_path=effective_athena_source, - tables=tables or None, - db_schema=conn.db_schema, - dry_run=dry_run, - merge_strategy=merge_strategy, - quote_mode=quote_mode, - chunksize=None if staging_chunk_size == 0 else staging_chunk_size, - bulk_mode=bulk_mode, - merge_batch_size=merge_batch_size, - progress_callback=_update_progress, - ) - progress.update( - task_id, completed=100.0, description="Athena vocabulary load complete" - ) + report = load_vocab_source( + engine, + vocab_engine=vocab_engine, + vocab_schema=conn.resolved.vocab_schema, + source_path=effective_athena_source, + tables=tables or None, + db_schema=conn.resolved.schema_name, + dry_run=dry_run, + merge_strategy=merge_strategy, + quote_mode=quote_mode, + chunksize=None if staging_chunk_size == 0 else staging_chunk_size, + bulk_mode=bulk_mode, + merge_batch_size=merge_batch_size, + progress_callback=_update_progress, + ) + progress.update( + task_id, completed=100.0, description="Athena vocabulary load complete" + ) + finally: + if vocab_engine is not engine: + vocab_engine.dispose() console.print(render_vocab_load_results(report.results)) console.print(render_vocab_load_summary(report, dry_run=dry_run)) diff --git a/tests/test_schema_provenance_guard.py b/tests/test_schema_provenance_guard.py new file mode 100644 index 0000000..d74b734 --- /dev/null +++ b/tests/test_schema_provenance_guard.py @@ -0,0 +1,115 @@ +"""Schema-provenance guard wired into create_missing_tables(). + +Live-Postgres regression: proves the guard actually fires and prevents DDL +for a genuinely reconfigured schema, rather than passing by coincidence. + +pg_engine (unlike pg_db) is a real, committing engine, so every provenance +row this file's tests write is a genuine commit. No cleanup here yet, +pending a generic test-cleanup interface in oa-configurator's testing +module. +""" + +from __future__ import annotations + +import uuid + +import pytest +import sqlalchemy as sa +from oa_configurator import ResolvedCDMDatabase, ResolvedConnection, SchemaDriftError +from oa_configurator.testing import isolated_test_schema + +from omop_alchemy.maintenance.cli_schema_tables import create_missing_tables + +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + + +def _resolved(pg_engine, *, database_name: str, schema: str) -> ResolvedCDMDatabase: + url = pg_engine.url + connection = ResolvedConnection( + name="guard_test_conn", + url=url.render_as_string(hide_password=False), + safe_url=url.render_as_string(hide_password=True), + _engine_url=url, + ) + return ResolvedCDMDatabase( + name=database_name, + connection=connection, + schema_name=schema, + vocab_connection=connection, + vocab_schema=schema, + results_schema=schema, + ) + + +def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_engine): + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={None: schema_a, "vocab": schema_a, "results": schema_a} + ) + create_missing_tables( + engine_a, + db_schema=schema_a, + resolved=_resolved(pg_engine, database_name=database_name, schema=schema_a), + test_only=False, + ) + + engine_b = pg_engine.execution_options( + schema_translate_map={None: schema_b, "vocab": schema_b, "results": schema_b} + ) + with pytest.raises(SchemaDriftError): + create_missing_tables( + engine_b, + db_schema=schema_b, + resolved=_resolved(pg_engine, database_name=database_name, schema=schema_b), + test_only=False, + ) + + # The guard raises before create_all() runs: schema_b must still be empty. + with engine_b.connect() as conn: + assert sa.inspect(conn).get_table_names(schema=schema_b) == [] + + +def test_create_missing_tables_guard_succeeds_on_agreeing_schema(pg_engine): + database_name = f"guard_wiring_agree_db_{uuid.uuid4().hex[:8]}" + with isolated_test_schema(pg_engine, prefix="guard_wiring_agree") as schema: + engine = pg_engine.execution_options( + schema_translate_map={None: schema, "vocab": schema, "results": schema} + ) + resolved = _resolved(pg_engine, database_name=database_name, schema=schema) + create_missing_tables(engine, db_schema=schema, resolved=resolved, test_only=False) + # Second call, same resolved schema, nothing missing: must not raise. + results = create_missing_tables(engine, db_schema=schema, resolved=resolved, test_only=False) + assert results == [] + + +def test_create_missing_tables_test_only_bypasses_guard(pg_engine): + database_name = f"guard_wiring_testonly_db_{uuid.uuid4().hex[:8]}" + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_to_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_to_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={None: schema_a, "vocab": schema_a, "results": schema_a} + ) + create_missing_tables( + engine_a, + db_schema=schema_a, + resolved=_resolved(pg_engine, database_name=database_name, schema=schema_a), + test_only=True, + ) + engine_b = pg_engine.execution_options( + schema_translate_map={None: schema_b, "vocab": schema_b, "results": schema_b} + ) + # Must not raise: test_only=True short-circuits the guard entirely. + create_missing_tables( + engine_b, + db_schema=schema_b, + resolved=_resolved(pg_engine, database_name=database_name, schema=schema_b), + test_only=True, + ) + with engine_b.connect() as conn: + assert "person" in sa.inspect(conn).get_table_names(schema=schema_b) diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index 1167d51..639511e 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -1,6 +1,6 @@ import pytest -from oa_configurator.testing import DIALECT_PARAMS +from oa_configurator.testing import DIALECT_PARAMS, isolated_test_schema from omop_alchemy.backends.sqlite import SQLiteBackend from omop_alchemy.cdm.base.indexing import omop_index_name from omop_alchemy.maintenance.cli_indexes import manage_indexes @@ -13,24 +13,18 @@ @pytest.fixture(params=DIALECT_PARAMS) def reconcile_engine(request): - """Every OMOP table created and indexed/clustered, on both a real - Postgres backend and SQLite: index-rename detection is dialect-portable - logic, not SQLite-specific, so it's genuinely worth exercising against - both. Only the postgresql param ever requests pg_session, so the - sqlite param never needs a database. - - DIALECT_PARAMS carries each dialect's own mark plus `forked` directly - on the param value, so this still works correctly even though - request.getfixturevalue("pg_session") is a dynamic, runtime lookup - invisible to pytest's collection-time fixturenames computation (the - usual pg_db-in-fixturenames auto-detection can't see it). - - manage_indexes(enable=True) matters here specifically on Postgres: - create_missing_tables() alone creates tables and their indexes, but - never physically CLUSTERs them, so a fresh Postgres database reports - genuine drift (cluster status MISSING) without this step. On SQLite, - where CLUSTER doesn't exist, this call is a harmless no-op on the - clustering half and just re-confirms indexes already exist. + """Every OMOP table created, indexed, and clustered, on both Postgres and SQLite. + + manage_indexes(enable=True) is required on Postgres: create_missing_tables() + alone creates indexes but never physically CLUSTERs them, so a fresh + database would otherwise report false cluster drift. It's a harmless + no-op for clustering on SQLite. + + Notes + ----- + DIALECT_PARAMS marks each param directly, since the postgresql param's + dynamic request.getfixturevalue("pg_session") call is invisible to + pytest's usual fixturenames-based auto-detection. """ if request.param == "postgresql": engine = request.getfixturevalue("pg_session").get_bind() @@ -104,6 +98,97 @@ def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(reconcile assert person_result.issue_count == 1 +@pytest.mark.postgresql +@pytest.mark.db_dialect +def test_reconcile_schema_reports_relocated_when_table_found_in_another_schema(pg_engine): + """A table missing from its expected schema but physically present under + a different one reports RELOCATED, not a plain MISSING. + """ + with ( + isolated_test_schema(pg_engine, prefix="reconcile_relocated_a") as schema_a, + isolated_test_schema(pg_engine, prefix="reconcile_relocated_b") as schema_b, + ): + engine = pg_engine.execution_options( + schema_translate_map={None: schema_a, "vocab": schema_a, "results": schema_a} + ) + # vocabulary_included defaults to True: person's gender_concept_id FK + # targets a vocab table, so excluding vocab here would leave that FK + # unresolved and person itself blocked from creation. + create_missing_tables(engine, db_schema=schema_a) + with engine.begin() as connection: + connection.exec_driver_sql(f'ALTER TABLE "{schema_a}".person SET SCHEMA "{schema_b}"') + + report = reconcile_schema(engine, db_schema=schema_a) + + person_result = next(r for r in report.table_results if r.table_name == "person") + assert person_result.status == "relocated" + person_issue = next( + i for i in report.issues if i.table_name == "person" and i.component == "table" + ) + assert person_issue.status == "relocated" + # public may also legitimately carry a person table from an + # unrelated database/test, so assert schema_b is among the + # relocated schemas rather than the only one reported. + assert schema_b in person_issue.actual.split(", ") + assert is_blocking_issue(person_issue) + + +@pytest.mark.postgresql +@pytest.mark.db_dialect +def test_reconcile_schema_with_resolved_qualifies_each_table_to_its_own_role_schema(pg_engine): + """A clinical table, a vocab table, and a results table each live in a + genuinely different physical schema. Passing resolved must compare each + against its own schema, not one blanket db_schema value, or the vocab + and results tables report false drift here. + """ + from oa_configurator import ResolvedCDMDatabase, ResolvedConnection + + with ( + isolated_test_schema(pg_engine, prefix="reconcile_three_primary") as primary_schema, + isolated_test_schema(pg_engine, prefix="reconcile_three_vocab") as vocab_schema, + isolated_test_schema(pg_engine, prefix="reconcile_three_results") as results_schema, + ): + url = pg_engine.url + connection = ResolvedConnection( + name="reconcile_three_conn", + url=url.render_as_string(hide_password=False), + safe_url=url.render_as_string(hide_password=True), + _engine_url=url, + ) + resolved = ResolvedCDMDatabase( + name="reconcile_three_db", + connection=connection, + schema_name=primary_schema, + vocab_connection=connection, + vocab_schema=vocab_schema, + results_schema=results_schema, + ) + engine = pg_engine.execution_options( + schema_translate_map={ + None: primary_schema, "vocab": vocab_schema, "results": results_schema + } + ) + create_missing_tables( + engine, db_schema=primary_schema, resolved=resolved, test_only=True + ) + + report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) + + # cluster/index issues are excluded here. manage_indexes()/ + # cli_indexes.py's cluster commands have their own, separate + # cross-schema bug (found while writing this test, not yet + # investigated). This test only verifies that reconcile_schema + # resolves each table's own role schema instead of one blanket value. + checked_components = {"table", "column", "primary_key", "foreign_key"} + for table_name in ("person", "concept", "observation_period"): + issues = [ + issue + for issue in report.issues + if issue.table_name == table_name and issue.component in checked_components + ] + assert issues == [], (table_name, issues) + + def test_is_blocking_issue_excludes_renamed_only(): from omop_alchemy.maintenance.cli_schema_reconcile import ReconciliationIssue from omop_alchemy.maintenance._cli_utils import Status @@ -183,12 +268,12 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_tar """person's cluster target is the primary key's own index ("pk_person"), not a declared secondary index, unlike episode. The official OHDSI CDM DDL always clusters such tables on a separate, non-unique index instead - (e.g. "idx_person_id"): this must still report 'renamed', not 'mismatch', - and the same physical index must not *also* be flagged as an unexpected - plain index -- both are the same latent bug (the cluster target's - equivalence check assuming the PK's own uniqueness applies to whatever - physically serves as the cluster index, and not being shared with the - general index-diffing pass).""" + (e.g. "idx_person_id"). This must still report 'renamed', not + 'mismatch', and the same physical index must not also be flagged as an + unexpected plain index. Both are the same latent bug: the cluster + target's equivalence check assumes the PK's own uniqueness applies to + whatever physically serves as the cluster index, and isn't shared with + the general index-diffing pass.""" engine = fresh_reconcile_engine with engine.begin() as connection: connection.exec_driver_sql("CREATE INDEX idx_person_id ON person (person_id)") diff --git a/tests/test_schema_rectify_cli.py b/tests/test_schema_rectify_cli.py new file mode 100644 index 0000000..2244cd0 --- /dev/null +++ b/tests/test_schema_rectify_cli.py @@ -0,0 +1,149 @@ +"""Rectify CLI: acknowledge-schema-migration / drop-orphan-schema-tables. + +Live-Postgres regression against real StackConfig/Resolver plumbing, not +mocked, since these commands' whole point is writing/reading real database +state. Monkeypatches cli_schema.load_stack_config (the name as imported into +that module) to point at a scratch StackConfig built from pg_engine's own +connection, rather than touching the real on-disk config. +""" + +from __future__ import annotations + +import uuid + +import pytest +import sqlalchemy as sa +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator.domains.resources.sql import _provenance_schema_for, _schema_provenance_table, ensure_schema +from oa_configurator.testing import isolated_test_schema +from sqlalchemy.engine import make_url +from typer.testing import CliRunner + +from omop_alchemy.maintenance.cli import app + +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + +runner = CliRunner() + + +@pytest.fixture +def cli_stack(pg_engine, monkeypatch): + """A StackConfig with one real, test_only Postgres connection/database + entry, pointed at pg_engine's own live server, the same connection every + other Postgres test in this repo already uses.""" + url = make_url(pg_engine.url).render_as_string(hide_password=False) + conn_url = make_url(url) + stack = StackConfig.for_session( + connections={ + "cli_rectify_conn": ConnectionConfig( + dialect=conn_url.drivername, + host=conn_url.host, + port=conn_url.port, + user=conn_url.username, + password=conn_url.password, + database_name=conn_url.database, + test_only=True, + ) + }, + databases={ + "cli_rectify_db": CDMDatabaseConfig(connection="cli_rectify_conn", schema_name="public"), + }, + ) + monkeypatch.setattr("omop_alchemy.maintenance.cli_schema.load_stack_config", lambda: stack) + return stack + + +def _cleanup_provenance_row(pg_engine, *, database_name: str, role: str) -> None: + with pg_engine.begin() as conn: + ensure_schema(conn, _provenance_schema_for(conn)) + table = _schema_provenance_table(_provenance_schema_for(conn)) + table.create(bind=conn, checkfirst=True) + conn.execute( + table.delete().where(table.c.database_name == database_name, table.c.role == role) + ) + + +def test_reason_is_mandatory(cli_stack): + result = runner.invoke( + app, + ["acknowledge-schema-migration", "--database", "cli_rectify_db", "--new-schema", "whatever"], + ) + assert result.exit_code != 0 + assert "--reason" in result.output + + +def test_acknowledge_writes_a_provenance_row(cli_stack, pg_engine): + role = "vocab" + schema = f"cli_ack_{uuid.uuid4().hex[:8]}" + try: + result = runner.invoke( + app, + [ + "acknowledge-schema-migration", + "--database", "cli_rectify_db", + "--role", role, + "--new-schema", schema, + "--reason", "regression test", + ], + ) + assert result.exit_code == 0, result.stdout + assert "Acknowledged" in result.stdout + + with pg_engine.connect() as conn: + table = _schema_provenance_table(_provenance_schema_for(conn)) + row = conn.execute( + sa.select(table.c.resolved_schema, table.c.reason).where( + table.c.database_name == "cli_rectify_db", table.c.role == role + ) + ).first() + assert row is not None + assert row.resolved_schema == schema + assert row.reason == "regression test" + finally: + _cleanup_provenance_row(pg_engine, database_name="cli_rectify_db", role=role) + + +def test_drop_orphan_refuses_against_a_databases_own_current_schema(cli_stack): + result = runner.invoke( + app, + ["drop-orphan-schema-tables", "--database", "cli_rectify_db", "--schema", "public"], + ) + assert result.exit_code != 0 + assert "current schema target" in result.stdout + + +def test_drop_orphan_previews_without_confirm(cli_stack, pg_engine): + with isolated_test_schema(pg_engine, prefix="cli_drop_preview") as schema: + with pg_engine.begin() as conn: + conn.exec_driver_sql(f'CREATE TABLE "{schema}".orphaned (id int)') + + result = runner.invoke( + app, ["drop-orphan-schema-tables", "--database", "cli_rectify_db", "--schema", schema] + ) + assert result.exit_code == 0, result.stdout + assert "Would drop" in result.stdout + assert "Preview only" in result.stdout + + with pg_engine.connect() as conn: + assert sa.inspect(conn).has_table("orphaned", schema=schema) + + +def test_drop_orphan_confirm_actually_drops(cli_stack, pg_engine): + with isolated_test_schema(pg_engine, prefix="cli_drop_confirm") as schema: + with pg_engine.begin() as conn: + conn.exec_driver_sql(f'CREATE TABLE "{schema}".orphaned (id int)') + + result = runner.invoke( + app, + [ + "drop-orphan-schema-tables", + "--database", "cli_rectify_db", + "--schema", schema, + "--confirm", + ], + ) + assert result.exit_code == 0, result.stdout + assert "Dropped" in result.stdout + + with pg_engine.connect() as conn: + assert not sa.inspect(conn).has_table("orphaned", schema=schema) From 7cf5ad7345c8eec7fe5b457ac3d3b775fef3e7ac Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Fri, 4 Sep 2026 06:07:39 +0000 Subject: [PATCH 06/32] Get rid of test_only, have MAINTENANCE_SCHEMA at the right place, proper reset and clean-up of test databases, drop the default schema on the primary DB as no longer required --- omop_alchemy/config.py | 6 ++ omop_alchemy/maintenance/_cli_utils.py | 7 +- omop_alchemy/maintenance/cli_indexes.py | 3 +- omop_alchemy/maintenance/cli_schema.py | 1 - .../maintenance/cli_schema_reconcile.py | 21 ++-- omop_alchemy/maintenance/cli_schema_tables.py | 16 ++- tests/conftest.py | 54 ++++++++--- tests/test_config_driver.py | 6 +- tests/test_foreign_keys.py | 6 +- tests/test_fulltext.py | 6 +- tests/test_indexes.py | 25 ++--- tests/test_load_vocab_source.py | 4 +- tests/test_schema_provenance_guard.py | 92 ++++++------------ tests/test_schema_reconcile.py | 97 ++++++++++++------- tests/test_truncate_tables.py | 4 +- 15 files changed, 187 insertions(+), 161 deletions(-) diff --git a/omop_alchemy/config.py b/omop_alchemy/config.py index a4b33a9..b08ce60 100644 --- a/omop_alchemy/config.py +++ b/omop_alchemy/config.py @@ -12,8 +12,14 @@ ResolvedCDMDatabase, Role, load_stack_config, + register_reserved_schema, ) +# Guaranteed to be imported and registered if there is a config +MAINTENANCE_SCHEMA: str = "omop_alchemy_maintenance" + +register_reserved_schema(MAINTENANCE_SCHEMA, owner="omop_alchemy") + class OmopAlchemyConfig(PackageConfigBase): """oa-configurator config class for omop-alchemy, the CDM database owner. diff --git a/omop_alchemy/maintenance/_cli_utils.py b/omop_alchemy/maintenance/_cli_utils.py index 27c9b41..c99b757 100644 --- a/omop_alchemy/maintenance/_cli_utils.py +++ b/omop_alchemy/maintenance/_cli_utils.py @@ -9,7 +9,7 @@ from typing import Any, Callable, TypeVar import typer -from oa_configurator import ResolvedCDMDatabase, register_reserved_schema +from oa_configurator import ResolvedCDMDatabase from sqlalchemy.exc import SQLAlchemyError from .tables import TableCategory @@ -20,11 +20,6 @@ _F = TypeVar("_F", bound=Callable[..., Any]) -MAINTENANCE_SCHEMA: str = "omop_alchemy_maintenance" - -register_reserved_schema(MAINTENANCE_SCHEMA, owner="omop_alchemy") - - class Severity(StrEnum): """Coarse-grained outcome classification shared by every maintenance command's status vocabulary. diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index 4aece88..8b57343 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -15,7 +15,8 @@ from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY from ..backends import resolve_backend, backend_supports -from ._cli_utils import MAINTENANCE_SCHEMA, Status, dry_label, dry_status, omop_command +from ..config import MAINTENANCE_SCHEMA +from ._cli_utils import Status, dry_label, dry_status, omop_command from .tables import ( MaintenanceTable, TableCategory, diff --git a/omop_alchemy/maintenance/cli_schema.py b/omop_alchemy/maintenance/cli_schema.py index 2f33c1c..01e53dc 100644 --- a/omop_alchemy/maintenance/cli_schema.py +++ b/omop_alchemy/maintenance/cli_schema.py @@ -163,7 +163,6 @@ def create_missing_tables_command( vocabulary_included=vocabulary_included, dry_run=dry_run, resolved=conn.resolved, - test_only=conn.resolved.connection.test_only, ) finally: if vocab_engine is not engine: diff --git a/omop_alchemy/maintenance/cli_schema_reconcile.py b/omop_alchemy/maintenance/cli_schema_reconcile.py index 03ac9a7..f69bef6 100644 --- a/omop_alchemy/maintenance/cli_schema_reconcile.py +++ b/omop_alchemy/maintenance/cli_schema_reconcile.py @@ -112,17 +112,19 @@ def _schema_qualified_tables( if resolved is None and db_schema is None: return {id(table): table for table in Base.metadata.tables.values()} metadata = sa.MetaData() + + def _referred_schema(_table: sa.Table, _to_schema, _constraint, referred_schema: str | None): + # None means "unchanged" to to_metadata(); BLANK_SCHEMA is what actually clears a schema tag. + target = _effective_schema(resolved, _role_from_schema_tag(referred_schema), db_schema) + return target if target is not None else sa.BLANK_SCHEMA + return { id(table): table.to_metadata( metadata, # SQLAlchemy's own stub omits None from schema's declared type, # despite accepting and correctly handling it at runtime schema=_effective_schema(resolved, _role_from_schema_tag(table.schema), db_schema), # ty: ignore[invalid-argument-type] - referred_schema_fn=( - lambda _table, _to_schema, _constraint, referred_schema: _effective_schema( - resolved, _role_from_schema_tag(referred_schema), db_schema - ) - ), + referred_schema_fn=_referred_schema, ) for table in Base.metadata.tables.values() } @@ -361,14 +363,19 @@ def reconcile_schema( expected_fks = _expected_foreign_keys(expected_table) actual_fks = _actual_foreign_keys(inspector, maintenance_table.table_name, table_schema) + # Uses the unqualified table, since two role-tagged tables can collapse to + # the same schema (e.g. all-None on SQLite) and hide a genuine cross-role FK. + raw_expected_fks = _expected_foreign_keys(maintenance_table.table) for signature, constraint in expected_fks.items(): if signature not in actual_fks: + raw_constraint = raw_expected_fks.get(signature) if ( not _cross_schema_fk_supported - and constraint.referred_table.schema != expected_table.schema + and raw_constraint is not None + and _role_from_schema_tag(raw_constraint.referred_table.schema) != table_role ): - # SQLite can never create an inline FK crossing a schema boundary + # SQLite can never create an inline FK crossing a schema boundary. continue constrained_columns, referred_table, referred_columns = signature table_issues.append( diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index 44316b3..ad56232 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -62,7 +62,6 @@ def create_missing_tables( vocabulary_included: bool = True, dry_run: bool = False, resolved: ResolvedCDMDatabase | None = None, - test_only: bool = False, ) -> list[TableCreationResult]: """Create any ORM-managed tables missing from the target database. Skips tables with unresolved FK dependencies. @@ -76,9 +75,8 @@ def create_missing_tables( Enables the schema-provenance guard around each ``create_all()`` call. Omitted by direct test/programmatic callers that hand in a bare engine with no resolved config behind it, in which case the - guard no-ops. - test_only : bool, optional - Forwarded to the guard. Ignored when *resolved* is None. + guard no-ops. A role whose connection is test_only=true also + no-ops, at the guard's own discretion. """ vocab_engine = vocab_engine if vocab_engine is not None else engine if not dry_run: @@ -116,8 +114,8 @@ def create_missing_tables( # One call: create_all's dependency sort and FK-deferral must see every table together. with ( engine.begin() as connection, - guard_schema_provenance(connection, resolved, role=Role.PRIMARY, test_only=test_only), - guard_schema_provenance(connection, resolved, role=Role.RESULTS, test_only=test_only), + guard_schema_provenance(connection, resolved, role=Role.PRIMARY), + guard_schema_provenance(connection, resolved, role=Role.RESULTS), ): Base.metadata.create_all( bind=connection, tables=all_tables, checkfirst=True @@ -134,8 +132,8 @@ def create_missing_tables( if other_tables: with ( engine.begin() as connection, - guard_schema_provenance(connection, resolved, role=Role.PRIMARY, test_only=test_only), - guard_schema_provenance(connection, resolved, role=Role.RESULTS, test_only=test_only), + guard_schema_provenance(connection, resolved, role=Role.PRIMARY), + guard_schema_provenance(connection, resolved, role=Role.RESULTS), ): Base.metadata.create_all( bind=connection, tables=other_tables, checkfirst=True @@ -143,7 +141,7 @@ def create_missing_tables( if vocab_tables: with ( vocab_engine.begin() as vocab_connection, - guard_schema_provenance(vocab_connection, resolved, role=Role.VOCAB, test_only=test_only), + guard_schema_provenance(vocab_connection, resolved, role=Role.VOCAB), ): Base.metadata.create_all( bind=vocab_connection, tables=vocab_tables, checkfirst=True diff --git a/tests/conftest.py b/tests/conftest.py index 9770870..d31e314 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -393,25 +393,51 @@ def pg_engine(pg_db): ) +_SYSTEM_SCHEMAS = frozenset({"pg_catalog", "information_schema"}) + + +def _reset_test_database(engine: sa.Engine) -> None: + """Drop every non-system schema and recreate public. + + Local to this file, not a shared oa-configurator primitive: unlike + every other package's committing-engine tests (self-cleaning via + isolated_test_schema()), pg_session's literal-"public"-name dependents + (see its own docstring) need a full reset, not just one schema's worth. + Assumes this test database isn't shared with a concurrently-running + process, the same assumption pg_session's public-only reset already + made. + """ + with engine.connect() as conn: + schema_names = sa.inspect(conn).get_schema_names() + for schema in schema_names: + if schema in _SYSTEM_SCHEMAS or schema.startswith("pg_"): + continue + conn.execute(sa.text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')) + conn.execute(sa.text("DROP SCHEMA IF EXISTS public CASCADE")) + conn.execute(sa.text("CREATE SCHEMA public")) + conn.commit() + + @pytest.fixture -def pg_session(pg_engine): +def pg_session(pg_engine, cleanup_after_test): """ Function-scoped PostgreSQL session with a clean schema for each test. - Drops and recreates the public schema before each test to ensure full - isolation. Cannot move onto ``isolated_test_schema()``'s real, - uniquely-named schema: some tests request ``pg_session`` and - ``pg_engine`` together and rely on both pointing at the same physical - schema (``pg_engine`` itself has no schema of its own, only whatever - the connection's own default is). Confirmed by a real failure when - this migration was tried: ``test_load_vocab_postgres.py``'s - ``pg_session, pg_engine`` tests broke, since ``pg_engine``-only calls - then targeted an unbootstrapped schema. + Resets every non-system schema (not just public) both before and after + each test, via cleanup_after_test, so a test's own committed DDL/DML + -- in public or a reserved bookkeeping schema like MAINTENANCE_SCHEMA + -- never depends on some later, unrelated test to wipe it. Cannot move + onto ``isolated_test_schema()``'s real, uniquely-named schema: some + tests request ``pg_session`` and ``pg_engine`` together and rely on + both pointing at the same physical schema (``pg_engine`` itself has no + schema of its own, only whatever the connection's own default is). + Confirmed by a real failure when this migration was tried: + ``test_load_vocab_postgres.py``'s ``pg_session, pg_engine`` tests + broke, since ``pg_engine``-only calls then targeted an unbootstrapped + schema. """ - with pg_engine.connect() as conn: - conn.execute(sa.text("DROP SCHEMA public CASCADE")) - conn.execute(sa.text("CREATE SCHEMA public")) - conn.commit() + _reset_test_database(pg_engine) + cleanup_after_test(lambda: _reset_test_database(pg_engine)) bootstrap(pg_engine, create=True) diff --git a/tests/test_config_driver.py b/tests/test_config_driver.py index c402842..fdaefad 100644 --- a/tests/test_config_driver.py +++ b/tests/test_config_driver.py @@ -51,9 +51,7 @@ def test_get_cdm_context_resolves_the_typed_database_field(monkeypatch) -> None: connections={ "cdm": ConnectionConfig(dialect="sqlite", database_name=":memory:") }, - databases={ - "cdm_db": CDMDatabaseConfig(connection="cdm", schema_name="main") - }, + databases={"cdm_db": CDMDatabaseConfig(connection="cdm")}, ) monkeypatch.setattr("omop_alchemy.config.load_stack_config", lambda: stack) @@ -61,7 +59,7 @@ def test_get_cdm_context_resolves_the_typed_database_field(monkeypatch) -> None: assert package_config.cdm_db == "cdm_db" assert isinstance(resolved, ResolvedCDMDatabase) - assert resolved.schema_name == "main" + assert resolved.schema_name is None def test_vocabulary_identity_for_colocated_vocabulary() -> None: diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 272c249..8797792 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -68,7 +68,7 @@ def test_disable_foreign_keys_cli_fails_gracefully_for_sqlite(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -242,7 +242,7 @@ def test_enable_foreign_keys_strict_cli_invokes_strict_management(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -354,7 +354,7 @@ def test_foreign_keys_validate_cli_invokes_validation(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", diff --git a/tests/test_fulltext.py b/tests/test_fulltext.py index 4a121de..0ff2b76 100644 --- a/tests/test_fulltext.py +++ b/tests/test_fulltext.py @@ -210,7 +210,11 @@ def test_fulltext_install_cli_passes_options(monkeypatch): calls: dict[str, object] = {} cfg = StackConfig.for_session( - connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, + connections={ + "db": ConnectionConfig( + dialect="postgresql+psycopg", host="localhost", database_name="db" + ) + }, databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="public")}, ) monkeypatch.setattr( diff --git a/tests/test_indexes.py b/tests/test_indexes.py index fea04d7..35570f7 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -1,14 +1,16 @@ import pytest import sqlalchemy as sa +from pydantic import ValidationError from typer.testing import CliRunner -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Resolver, StackConfig, qualified +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig, qualified from oa_configurator.testing import DIALECT_PARAMS from omop_alchemy.backends.sqlite import SQLiteBackend from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY, omop_index_name from omop_alchemy.maintenance.cli import app from omop_alchemy.maintenance.cli_schema import create_missing_tables -from omop_alchemy.maintenance._cli_utils import MAINTENANCE_SCHEMA, Status +from omop_alchemy.config import MAINTENANCE_SCHEMA +from omop_alchemy.maintenance._cli_utils import Status from omop_alchemy.maintenance.ui import render_index_summary from omop_alchemy.maintenance.cli_indexes import ( IndexManagementResult, @@ -366,7 +368,7 @@ def test_disable_indexes_cli_invokes_management(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -430,7 +432,7 @@ def test_enable_indexes_cli_no_cluster_flag_passes_through(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -564,14 +566,13 @@ def test_describe_shape_conflict_mentions_reason(): def test_resolving_cdm_database_with_maintenance_schema_name_raises(): - cfg = StackConfig.for_session( - connections={"c": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={ - "default": CDMDatabaseConfig(connection="c", schema_name=MAINTENANCE_SCHEMA) - }, - ) - with pytest.raises(RuntimeError, match=f"{MAINTENANCE_SCHEMA!r}.*omop_alchemy"): - Resolver(cfg).resolve_database("default") + with pytest.raises(ValidationError, match=f"{MAINTENANCE_SCHEMA!r}.*omop_alchemy"): + StackConfig.for_session( + connections={"c": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, + databases={ + "default": CDMDatabaseConfig(connection="c", schema_name=MAINTENANCE_SCHEMA) + }, + ) # ── Foreign-named equivalent index reconciliation ──────────────────────────────── diff --git a/tests/test_load_vocab_source.py b/tests/test_load_vocab_source.py index 33dfb35..6975087 100644 --- a/tests/test_load_vocab_source.py +++ b/tests/test_load_vocab_source.py @@ -177,7 +177,7 @@ def test_load_vocab_source_cli_uses_configured_athena_source(monkeypatch, tmp_pa connections={ "db": ConnectionConfig(dialect="sqlite", database_name=":memory:") }, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, tools={OmopAlchemyConfig.tool_name: {"athena_source_path": str(athena_dir)}}, ) @@ -481,7 +481,7 @@ def test_load_vocab_source_cli_surfaces_database_error_detail(monkeypatch): connections={ "db": ConnectionConfig(dialect="sqlite", database_name=":memory:") }, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", diff --git a/tests/test_schema_provenance_guard.py b/tests/test_schema_provenance_guard.py index d74b734..c7e3e34 100644 --- a/tests/test_schema_provenance_guard.py +++ b/tests/test_schema_provenance_guard.py @@ -3,46 +3,56 @@ Live-Postgres regression: proves the guard actually fires and prevents DDL for a genuinely reconfigured schema, rather than passing by coincidence. +Only that one case is covered here. The guard's own agree/no-op/test_only +semantics are already exhaustively covered at the primitive level in +oa-configurator's own test suite; what's worth proving per consuming repo +is that this call site is actually wired to it, and a wiring mistake would +show up here too. + pg_engine (unlike pg_db) is a real, committing engine, so every provenance -row this file's tests write is a genuine commit. No cleanup here yet, -pending a generic test-cleanup interface in oa-configurator's testing -module. +row this file's tests write is a genuine commit. cleanup_after_test deletes +this test's own schema_provenance rows at teardown (see Phase 10.12 in the +plan). """ from __future__ import annotations +import dataclasses import uuid import pytest import sqlalchemy as sa -from oa_configurator import ResolvedCDMDatabase, ResolvedConnection, SchemaDriftError -from oa_configurator.testing import isolated_test_schema +from oa_configurator import SchemaDriftError +from oa_configurator.domains.resources.sql import SCHEMA_PROVENANCE_SCHEMA, _schema_provenance_table + +from oa_configurator.testing import delete_rows_on_cleanup, isolated_test_schema from omop_alchemy.maintenance.cli_schema_tables import create_missing_tables pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] -def _resolved(pg_engine, *, database_name: str, schema: str) -> ResolvedCDMDatabase: - url = pg_engine.url - connection = ResolvedConnection( - name="guard_test_conn", - url=url.render_as_string(hide_password=False), - safe_url=url.render_as_string(hide_password=True), - _engine_url=url, - ) - return ResolvedCDMDatabase( +def _resolved(pg_db, *, database_name: str, schema: str): + """pg_db.resolved with a unique name (the guard's own key includes it), + all three schemas pointed at schema, and connection.test_only forced + False so the guard doesn't no-op against pg_db's own test-only marking. + """ + return dataclasses.replace( + pg_db.resolved, name=database_name, - connection=connection, schema_name=schema, - vocab_connection=connection, vocab_schema=schema, results_schema=schema, + connection=dataclasses.replace(pg_db.resolved.connection, test_only=False), ) -def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_engine): +def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) with ( isolated_test_schema(pg_engine, prefix="guard_wiring_a") as schema_a, isolated_test_schema(pg_engine, prefix="guard_wiring_b") as schema_b, @@ -53,8 +63,7 @@ def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_engine): create_missing_tables( engine_a, db_schema=schema_a, - resolved=_resolved(pg_engine, database_name=database_name, schema=schema_a), - test_only=False, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_a), ) engine_b = pg_engine.execution_options( @@ -64,52 +73,9 @@ def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_engine): create_missing_tables( engine_b, db_schema=schema_b, - resolved=_resolved(pg_engine, database_name=database_name, schema=schema_b), - test_only=False, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), ) # The guard raises before create_all() runs: schema_b must still be empty. with engine_b.connect() as conn: assert sa.inspect(conn).get_table_names(schema=schema_b) == [] - - -def test_create_missing_tables_guard_succeeds_on_agreeing_schema(pg_engine): - database_name = f"guard_wiring_agree_db_{uuid.uuid4().hex[:8]}" - with isolated_test_schema(pg_engine, prefix="guard_wiring_agree") as schema: - engine = pg_engine.execution_options( - schema_translate_map={None: schema, "vocab": schema, "results": schema} - ) - resolved = _resolved(pg_engine, database_name=database_name, schema=schema) - create_missing_tables(engine, db_schema=schema, resolved=resolved, test_only=False) - # Second call, same resolved schema, nothing missing: must not raise. - results = create_missing_tables(engine, db_schema=schema, resolved=resolved, test_only=False) - assert results == [] - - -def test_create_missing_tables_test_only_bypasses_guard(pg_engine): - database_name = f"guard_wiring_testonly_db_{uuid.uuid4().hex[:8]}" - with ( - isolated_test_schema(pg_engine, prefix="guard_wiring_to_a") as schema_a, - isolated_test_schema(pg_engine, prefix="guard_wiring_to_b") as schema_b, - ): - engine_a = pg_engine.execution_options( - schema_translate_map={None: schema_a, "vocab": schema_a, "results": schema_a} - ) - create_missing_tables( - engine_a, - db_schema=schema_a, - resolved=_resolved(pg_engine, database_name=database_name, schema=schema_a), - test_only=True, - ) - engine_b = pg_engine.execution_options( - schema_translate_map={None: schema_b, "vocab": schema_b, "results": schema_b} - ) - # Must not raise: test_only=True short-circuits the guard entirely. - create_missing_tables( - engine_b, - db_schema=schema_b, - resolved=_resolved(pg_engine, database_name=database_name, schema=schema_b), - test_only=True, - ) - with engine_b.connect() as conn: - assert "person" in sa.inspect(conn).get_table_names(schema=schema_b) diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index 639511e..1832383 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -1,5 +1,10 @@ +import dataclasses +from typing import NamedTuple + import pytest +import sqlalchemy as sa +from oa_configurator import ResolvedCDMDatabase, ResolvedConnection from oa_configurator.testing import DIALECT_PARAMS, isolated_test_schema from omop_alchemy.backends.sqlite import SQLiteBackend from omop_alchemy.cdm.base.indexing import omop_index_name @@ -11,8 +16,36 @@ EPISODE_PERSON_INDEX = omop_index_name("episode", "person_id") +class _ReconcileEngine(NamedTuple): + engine: sa.Engine + resolved: ResolvedCDMDatabase + + +def _sqlite_resolved(engine: sa.Engine) -> ResolvedCDMDatabase: + """A ResolvedCDMDatabase for engine, single-schema (None throughout) to + match fresh_engine's own forced schema_translate_map. SQLite has no real + connection identity to resolve, so building this directly from the + engine's own URL is the whole story, unlike Postgres. + """ + url = engine.url + connection = ResolvedConnection( + name="fresh_engine_test", + url=url.render_as_string(hide_password=False), + safe_url=url.render_as_string(hide_password=True), + _engine_url=url, + ) + return ResolvedCDMDatabase( + name="fresh_engine_test", + connection=connection, + schema_name=None, + vocab_connection=connection, + vocab_schema=None, + results_schema=None, + ) + + @pytest.fixture(params=DIALECT_PARAMS) -def reconcile_engine(request): +def reconcile_engine(request) -> _ReconcileEngine: """Every OMOP table created, indexed, and clustered, on both Postgres and SQLite. manage_indexes(enable=True) is required on Postgres: create_missing_tables() @@ -27,16 +60,18 @@ def reconcile_engine(request): pytest's usual fixturenames-based auto-detection. """ if request.param == "postgresql": + resolved = request.getfixturevalue("pg_db").resolved engine = request.getfixturevalue("pg_session").get_bind() else: engine = request.getfixturevalue("fresh_engine") + resolved = _sqlite_resolved(engine) create_missing_tables(engine) manage_indexes(engine, enable=True) - return engine + return _ReconcileEngine(engine, resolved) @pytest.fixture -def fresh_reconcile_engine(fresh_engine): +def fresh_reconcile_engine(fresh_engine) -> _ReconcileEngine: """fresh_engine with every OMOP table already created. SQLite-only, unlike reconcile_engine above: the tests using this @@ -47,7 +82,7 @@ def fresh_reconcile_engine(fresh_engine): call, not a mock swap, so they stay a separate, SQLite-specific fixture. """ create_missing_tables(fresh_engine) - return fresh_engine + return _ReconcileEngine(fresh_engine, _sqlite_resolved(fresh_engine)) def _person_gender_issues(report): @@ -61,8 +96,8 @@ def _person_gender_issues(report): def test_reconcile_schema_reports_no_drift_on_fresh_database(reconcile_engine): - engine = reconcile_engine - report = reconcile_schema(engine) + engine, resolved = reconcile_engine + report = reconcile_schema(engine, resolved=resolved) person_result = next(r for r in report.table_results if r.table_name == "person") assert person_result.status == "matched" @@ -70,12 +105,12 @@ def test_reconcile_schema_reports_no_drift_on_fresh_database(reconcile_engine): def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(reconcile_engine): - engine = reconcile_engine + engine, resolved = reconcile_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") connection.exec_driver_sql("CREATE INDEX idx_gender ON person (gender_concept_id)") - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) issues = _person_gender_issues(report) assert len(issues) == 1 @@ -86,12 +121,12 @@ def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(rec def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(reconcile_engine): - engine = reconcile_engine + engine, resolved = reconcile_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") connection.exec_driver_sql("CREATE INDEX idx_gender ON person (gender_concept_id)") - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) person_result = next(r for r in report.table_results if r.table_name == "person") assert person_result.status == "matched" @@ -100,7 +135,7 @@ def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(reconcile @pytest.mark.postgresql @pytest.mark.db_dialect -def test_reconcile_schema_reports_relocated_when_table_found_in_another_schema(pg_engine): +def test_reconcile_schema_reports_relocated_when_table_found_in_another_schema(pg_db, pg_engine): """A table missing from its expected schema but physically present under a different one reports RELOCATED, not a plain MISSING. """ @@ -108,17 +143,20 @@ def test_reconcile_schema_reports_relocated_when_table_found_in_another_schema(p isolated_test_schema(pg_engine, prefix="reconcile_relocated_a") as schema_a, isolated_test_schema(pg_engine, prefix="reconcile_relocated_b") as schema_b, ): + resolved = dataclasses.replace( + pg_db.resolved, schema_name=schema_a, vocab_schema=schema_a, results_schema=schema_a + ) engine = pg_engine.execution_options( schema_translate_map={None: schema_a, "vocab": schema_a, "results": schema_a} ) # vocabulary_included defaults to True: person's gender_concept_id FK # targets a vocab table, so excluding vocab here would leave that FK # unresolved and person itself blocked from creation. - create_missing_tables(engine, db_schema=schema_a) + create_missing_tables(engine, db_schema=schema_a, resolved=resolved) with engine.begin() as connection: connection.exec_driver_sql(f'ALTER TABLE "{schema_a}".person SET SCHEMA "{schema_b}"') - report = reconcile_schema(engine, db_schema=schema_a) + report = reconcile_schema(engine, resolved=resolved) person_result = next(r for r in report.table_results if r.table_name == "person") assert person_result.status == "relocated" @@ -135,31 +173,20 @@ def test_reconcile_schema_reports_relocated_when_table_found_in_another_schema(p @pytest.mark.postgresql @pytest.mark.db_dialect -def test_reconcile_schema_with_resolved_qualifies_each_table_to_its_own_role_schema(pg_engine): +def test_reconcile_schema_with_resolved_qualifies_each_table_to_its_own_role_schema(pg_db, pg_engine): """A clinical table, a vocab table, and a results table each live in a genuinely different physical schema. Passing resolved must compare each against its own schema, not one blanket db_schema value, or the vocab and results tables report false drift here. """ - from oa_configurator import ResolvedCDMDatabase, ResolvedConnection - with ( isolated_test_schema(pg_engine, prefix="reconcile_three_primary") as primary_schema, isolated_test_schema(pg_engine, prefix="reconcile_three_vocab") as vocab_schema, isolated_test_schema(pg_engine, prefix="reconcile_three_results") as results_schema, ): - url = pg_engine.url - connection = ResolvedConnection( - name="reconcile_three_conn", - url=url.render_as_string(hide_password=False), - safe_url=url.render_as_string(hide_password=True), - _engine_url=url, - ) - resolved = ResolvedCDMDatabase( - name="reconcile_three_db", - connection=connection, + resolved = dataclasses.replace( + pg_db.resolved, schema_name=primary_schema, - vocab_connection=connection, vocab_schema=vocab_schema, results_schema=results_schema, ) @@ -168,9 +195,7 @@ def test_reconcile_schema_with_resolved_qualifies_each_table_to_its_own_role_sch None: primary_schema, "vocab": vocab_schema, "results": results_schema } ) - create_missing_tables( - engine, db_schema=primary_schema, resolved=resolved, test_only=True - ) + create_missing_tables(engine, db_schema=primary_schema, resolved=resolved) report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) @@ -212,7 +237,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_inde """A table physically clustered on a foreign-named equivalent of the ORM's cluster index (e.g. captured/restored under its original name by manage_indexes()) must report a 'renamed' cluster issue, not 'mismatch'.""" - engine = fresh_reconcile_engine + engine, resolved = fresh_reconcile_engine with engine.begin() as connection: connection.exec_driver_sql(f"DROP INDEX {EPISODE_PERSON_INDEX}") connection.exec_driver_sql("CREATE INDEX idx_episode_person ON episode (person_id)") @@ -225,7 +250,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_inde ), ) - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) episode_result = next(r for r in report.table_results if r.table_name == "episode") cluster_issues = [ issue for issue in report.issues @@ -242,7 +267,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_inde def test_reconcile_schema_cluster_check_still_reports_real_mismatch(fresh_reconcile_engine, monkeypatch): """A genuinely different physical cluster state (not just a foreign-named equivalent) must still be reported as drift.""" - engine = fresh_reconcile_engine + engine, resolved = fresh_reconcile_engine monkeypatch.setattr( SQLiteBackend, @@ -252,7 +277,7 @@ def test_reconcile_schema_cluster_check_still_reports_real_mismatch(fresh_reconc ), ) - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) episode_result = next(r for r in report.table_results if r.table_name == "episode") cluster_issues = [ issue for issue in report.issues @@ -274,7 +299,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_tar target's equivalence check assumes the PK's own uniqueness applies to whatever physically serves as the cluster index, and isn't shared with the general index-diffing pass.""" - engine = fresh_reconcile_engine + engine, resolved = fresh_reconcile_engine with engine.begin() as connection: connection.exec_driver_sql("CREATE INDEX idx_person_id ON person (person_id)") @@ -286,7 +311,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_tar ), ) - report = reconcile_schema(engine) + report = reconcile_schema(engine, resolved=resolved) person_result = next(r for r in report.table_results if r.table_name == "person") person_issues = [issue for issue in report.issues if issue.table_name == "person"] cluster_issues = [issue for issue in person_issues if issue.component == "cluster"] diff --git a/tests/test_truncate_tables.py b/tests/test_truncate_tables.py index 5c4f3c7..52eb151 100644 --- a/tests/test_truncate_tables.py +++ b/tests/test_truncate_tables.py @@ -44,7 +44,7 @@ def test_truncate_tables_cli_requires_confirmation(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", @@ -63,7 +63,7 @@ def test_truncate_tables_cli_invokes_management(monkeypatch): cfg = StackConfig.for_session( connections={"db": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="main")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", From 8ef98f1ebe1caf2e67815824684efaace136dbd9 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Mon, 7 Sep 2026 05:16:00 +0000 Subject: [PATCH 07/32] Merge the move of dialects to oa-configurator --- omop_alchemy/backends/__init__.py | 3 +- omop_alchemy/backends/postgres.py | 4 +-- omop_alchemy/backends/resolve.py | 25 +++++++------- omop_alchemy/backends/sqlite.py | 3 +- omop_alchemy/maintenance/cli_schema_doctor.py | 12 +++---- omop_alchemy/maintenance/cli_schema_info.py | 34 +++++++------------ omop_alchemy/maintenance/cli_vocab.py | 7 ++-- omop_alchemy/maintenance/ui.py | 16 +++------ tests/test_schema_reconcile.py | 28 +++++++++++---- 9 files changed, 66 insertions(+), 66 deletions(-) diff --git a/omop_alchemy/backends/__init__.py b/omop_alchemy/backends/__init__.py index f0f980e..78934f0 100644 --- a/omop_alchemy/backends/__init__.py +++ b/omop_alchemy/backends/__init__.py @@ -12,7 +12,7 @@ ) from .postgres import PostgresBackend from .sqlite import SQLiteBackend -from .resolve import resolve_backend, SupportedDialect +from .resolve import resolve_backend __all__ = [ "Backend", @@ -28,5 +28,4 @@ "PostgresBackend", "SQLiteBackend", "resolve_backend", - "SupportedDialect", ] diff --git a/omop_alchemy/backends/postgres.py b/omop_alchemy/backends/postgres.py index c1d6d0d..6a964f9 100644 --- a/omop_alchemy/backends/postgres.py +++ b/omop_alchemy/backends/postgres.py @@ -5,7 +5,7 @@ import sqlalchemy as sa -from oa_configurator import qualified, schema_of +from oa_configurator import Dialect, qualified, schema_of from sqlalchemy.dialects.postgresql import REGCONFIG, TSVECTOR from sqlalchemy.sql import func @@ -20,7 +20,7 @@ def name(self) -> str: @property def dialect(self) -> str: - return "postgresql" + return Dialect.POSTGRESQL # ── FK trigger management ──────────────────────────────────────────────── diff --git a/omop_alchemy/backends/resolve.py b/omop_alchemy/backends/resolve.py index f69e526..bc84e50 100644 --- a/omop_alchemy/backends/resolve.py +++ b/omop_alchemy/backends/resolve.py @@ -1,32 +1,33 @@ from __future__ import annotations -from enum import StrEnum import sqlalchemy as sa +from oa_configurator import Dialect from .base import Backend, BackendNotSupportedError from .postgres import PostgresBackend from .sqlite import SQLiteBackend - -class SupportedDialect(StrEnum): - POSTGRESQL = "postgresql" - SQLITE = "sqlite" - -_DIALECT_TO_BACKEND_MAP: dict[SupportedDialect, Backend] = { - SupportedDialect.POSTGRESQL: PostgresBackend(), - SupportedDialect.SQLITE: SQLiteBackend(), +_DIALECT_TO_BACKEND_MAP: dict[Dialect, Backend] = { + Dialect.POSTGRESQL: PostgresBackend(), + Dialect.SQLITE: SQLiteBackend(), } def resolve_backend(engine: sa.Engine) -> Backend: dialect = engine.dialect.name try: - supported_dialect = SupportedDialect(dialect) + supported_dialect = Dialect(dialect) except ValueError: raise BackendNotSupportedError( f"Unsupported database dialect: '{dialect}'. " - f"Supported dialects: {', '.join(sorted(SupportedDialect))}." + f"Supported dialects: {', '.join(sorted(Dialect))}." ) return _DIALECT_TO_BACKEND_MAP[supported_dialect] - +def backend_label(dialect_name: str) -> str: + """Human-readable backend name for a dialect, falling back to the raw + dialect name for anything unrecognized (e.g. connection not yet resolved).""" + try: + return _DIALECT_TO_BACKEND_MAP[Dialect(dialect_name)].name + except (ValueError, KeyError): + return dialect_name diff --git a/omop_alchemy/backends/sqlite.py b/omop_alchemy/backends/sqlite.py index b26fedd..1803c8b 100644 --- a/omop_alchemy/backends/sqlite.py +++ b/omop_alchemy/backends/sqlite.py @@ -1,6 +1,7 @@ from __future__ import annotations import sqlalchemy as sa +from oa_configurator import Dialect from .base import Backend, FeatureNotSupportedError @@ -13,7 +14,7 @@ def name(self) -> str: @property def dialect(self) -> str: - return "sqlite" + return Dialect.SQLITE def index_exists( self, diff --git a/omop_alchemy/maintenance/cli_schema_doctor.py b/omop_alchemy/maintenance/cli_schema_doctor.py index 010dcd8..cb0948a 100644 --- a/omop_alchemy/maintenance/cli_schema_doctor.py +++ b/omop_alchemy/maintenance/cli_schema_doctor.py @@ -5,9 +5,7 @@ from dataclasses import dataclass import sqlalchemy as sa -from oa_configurator import ResolvedDatabase - -from omop_alchemy.backends.resolve import SupportedDialect +from oa_configurator import Dialect, ResolvedDatabase from ._cli_utils import Status from .cli_foreign_keys import ( @@ -141,7 +139,7 @@ def _build_recommendations( ) ) - if info.backend == SupportedDialect.POSTGRESQL and info.pg_dump_path is None: + if info.backend == Dialect.POSTGRESQL and info.pg_dump_path is None: recommendations.append( DoctorRecommendation( status=Status.WARNING, @@ -151,7 +149,7 @@ def _build_recommendations( ) if ( - info.backend == SupportedDialect.POSTGRESQL + info.backend == Dialect.POSTGRESQL and info.pg_restore_path is None and info.psql_path is None ): @@ -273,7 +271,7 @@ def collect_doctor_report( ) ) - if info.backend == SupportedDialect.POSTGRESQL: + if info.backend == Dialect.POSTGRESQL: foreign_key_status = tuple( collect_foreign_key_trigger_status( engine, @@ -372,7 +370,7 @@ def collect_doctor_report( ) ) - if info.backend == SupportedDialect.POSTGRESQL: + if info.backend == Dialect.POSTGRESQL: backup_tools_ready = info.pg_dump_path is not None and ( info.pg_restore_path is not None or info.psql_path is not None ) diff --git a/omop_alchemy/maintenance/cli_schema_info.py b/omop_alchemy/maintenance/cli_schema_info.py index 09cbe0a..722e628 100644 --- a/omop_alchemy/maintenance/cli_schema_info.py +++ b/omop_alchemy/maintenance/cli_schema_info.py @@ -10,11 +10,11 @@ import sqlalchemy as sa from sqlalchemy.exc import ArgumentError, SQLAlchemyError -from oa_configurator import ResolvedCDMDatabase, Resolver, load_stack_config +from oa_configurator import Dialect, ResolvedCDMDatabase, Resolver, load_stack_config from oa_configurator.loader import DEFAULT_CONFIG_PATH -from omop_alchemy.backends.resolve import SupportedDialect from omop_alchemy.config import OmopAlchemyConfig +from ..backends.resolve import backend_label from ._cli_utils import Status from .cli_schema_tables import collect_missing_tables from .tables import ( @@ -23,14 +23,6 @@ ) -def _backend_label(dialect_name: str) -> str: - from ..backends.resolve import _DIALECT_TO_BACKEND_MAP, SupportedDialect - try: - return _DIALECT_TO_BACKEND_MAP[SupportedDialect(dialect_name)].name - except (ValueError, KeyError): - return dialect_name - - # --------------------------------------------------------------------------- # info # --------------------------------------------------------------------------- @@ -146,7 +138,7 @@ def _command_support_for_backend( psql_path: str | None, ) -> tuple[CommandSupport, ...]: """Compute the readiness status of every CLI command given the current backend, connection state, and tool availability.""" - current_backend = _backend_label(backend) + current_backend = backend_label(backend) if not engine_created: blocked_detail = ( f"Backend resolved to {current_backend}, but the engine could not be created: {engine_error}" @@ -164,7 +156,7 @@ def _command_support_for_backend( f"Ready on {current_backend}." if connection_ready else blocked_detail ) - if backend == SupportedDialect.POSTGRESQL: + if backend == Dialect.POSTGRESQL: analyze_status = portable_status analyze_detail = ( "Ready on PostgreSQL; ANALYZE and VACUUM ANALYZE are both supported." @@ -185,7 +177,7 @@ def _command_support_for_backend( if connection_ready else blocked_detail ) - elif backend == "sqlite": + elif backend == Dialect.SQLITE: analyze_status = Status.LIMITED if connection_ready else Status.BLOCKED analyze_detail = ( "Ready on SQLite; ANALYZE is supported, but `--vacuum` is unavailable." @@ -250,18 +242,18 @@ def _command_support_for_backend( "PostgreSQL + pg_dump", ( Status.READY - if connection_ready and backend == SupportedDialect.POSTGRESQL and pg_dump_path is not None + if connection_ready and backend == Dialect.POSTGRESQL and pg_dump_path is not None else Status.BLOCKED - if backend == SupportedDialect.POSTGRESQL + if backend == Dialect.POSTGRESQL else Status.UNSUPPORTED if connection_ready else Status.BLOCKED ), ( "Ready on PostgreSQL; `pg_dump` is available." - if connection_ready and backend == SupportedDialect.POSTGRESQL and pg_dump_path is not None + if connection_ready and backend == Dialect.POSTGRESQL and pg_dump_path is not None else "PostgreSQL is configured, but `pg_dump` is not on PATH." - if connection_ready and backend == SupportedDialect.POSTGRESQL + if connection_ready and backend == Dialect.POSTGRESQL else f"Requires PostgreSQL. Current backend: {current_backend}." if connection_ready else blocked_detail @@ -272,18 +264,18 @@ def _command_support_for_backend( "PostgreSQL + pg_restore/psql", ( Status.READY - if connection_ready and backend == SupportedDialect.POSTGRESQL and (pg_restore_path is not None or psql_path is not None) + if connection_ready and backend == Dialect.POSTGRESQL and (pg_restore_path is not None or psql_path is not None) else Status.BLOCKED - if backend == SupportedDialect.POSTGRESQL + if backend == Dialect.POSTGRESQL else Status.UNSUPPORTED if connection_ready else Status.BLOCKED ), ( "Ready on PostgreSQL; restore client tooling is available." - if connection_ready and backend == SupportedDialect.POSTGRESQL and (pg_restore_path is not None or psql_path is not None) + if connection_ready and backend == Dialect.POSTGRESQL and (pg_restore_path is not None or psql_path is not None) else "PostgreSQL is configured, but neither `pg_restore` nor `psql` is on PATH." - if connection_ready and backend == SupportedDialect.POSTGRESQL + if connection_ready and backend == Dialect.POSTGRESQL else f"Requires PostgreSQL. Current backend: {current_backend}." if connection_ready else blocked_detail diff --git a/omop_alchemy/maintenance/cli_vocab.py b/omop_alchemy/maintenance/cli_vocab.py index 46886fa..9458535 100644 --- a/omop_alchemy/maintenance/cli_vocab.py +++ b/omop_alchemy/maintenance/cli_vocab.py @@ -12,7 +12,7 @@ import sqlalchemy.orm as so from sqlalchemy.exc import OperationalError import typer -from oa_configurator import ensure_schema +from oa_configurator import Dialect, ensure_schema from orm_loader.backends import STAGING_SCHEMA, resolve_backend from orm_loader.helpers import Base from orm_loader.tables.typing import CSVTableProtocol @@ -25,7 +25,6 @@ TimeElapsedColumn, ) -from ..backends.resolve import SupportedDialect from omop_alchemy.cdm.model.vocabulary import ( Concept, Concept_Ancestor, @@ -365,7 +364,7 @@ def load_vocab_source( ) _use_bulk_mode = ( - bulk_mode and not dry_run and engine.dialect.name == SupportedDialect.POSTGRESQL + bulk_mode and not dry_run and engine.dialect.name == Dialect.POSTGRESQL ) if _use_bulk_mode: _emit( @@ -569,7 +568,7 @@ def load_vocab_source( table_count=table_count, ) - if not dry_run and engine.dialect.name == SupportedDialect.POSTGRESQL: + if not dry_run and engine.dialect.name == Dialect.POSTGRESQL: sequence_results = reset_model_sequences( engine, db_schema=db_schema, diff --git a/omop_alchemy/maintenance/ui.py b/omop_alchemy/maintenance/ui.py index ce3364a..e0a2492 100644 --- a/omop_alchemy/maintenance/ui.py +++ b/omop_alchemy/maintenance/ui.py @@ -9,7 +9,7 @@ from rich.table import Table from rich.text import Text -from ..backends.resolve import _DIALECT_TO_BACKEND_MAP, SupportedDialect as _SupportedDialect +from ..backends.resolve import backend_label from .ascii import render_banner from .tables import TableCategory @@ -60,12 +60,6 @@ def _status_style(status: Status) -> str: ) return status.severity.style -def _backend_label(dialect_name: str) -> str: - try: - return _DIALECT_TO_BACKEND_MAP[_SupportedDialect(dialect_name)].name - except (ValueError, KeyError): - return dialect_name - def _bool_label(value: bool) -> Text: return Text("yes" if value else "no", style="green" if value else "dim") @@ -173,7 +167,7 @@ def render_info_database(info: MaintenanceInfo) -> Panel: grid.add_column(style="bold cyan") grid.add_column() grid.add_row("Engine URL", info.engine_url or "-") - grid.add_row("Backend", _backend_label(info.backend) if info.backend else "-") + grid.add_row("Backend", backend_label(info.backend) if info.backend else "-") grid.add_row("Engine created", _bool_label(info.engine_created)) grid.add_row("Connection ready", _bool_label(info.connection_ready)) @@ -215,7 +209,7 @@ def render_backup_result(result: BackupResult) -> Panel: grid.add_column(style="bold cyan") grid.add_column() grid.add_row("Status", _status_text(result.status)) - grid.add_row("Backend", _backend_label(result.backend)) + grid.add_row("Backend", backend_label(result.backend)) grid.add_row("Database", result.database_name) grid.add_row("Schema", result.schema_name or "all schemas") grid.add_row("Format", result.backup_format.value) @@ -249,7 +243,7 @@ def render_restore_result(result: BackupResult) -> Panel: grid.add_column(style="bold cyan") grid.add_column() grid.add_row("Status", _status_text(result.status)) - grid.add_row("Backend", _backend_label(result.backend)) + grid.add_row("Backend", backend_label(result.backend)) grid.add_row("Database", result.database_name) grid.add_row("Schema", result.schema_name or "all schemas") grid.add_row("Format", result.backup_format.value) @@ -348,7 +342,7 @@ def render_reconciliation_summary(report: SchemaReconciliationReport) -> Panel: grid = Table.grid(padding=(0, 2)) grid.add_column(style="bold cyan") grid.add_column() - grid.add_row("Backend", _backend_label(report.backend)) + grid.add_row("Backend", backend_label(report.backend)) grid.add_row("Tables", str(len(report.table_results))) if matched: grid.add_row(Status.MATCHED.value.capitalize(), str(matched)) diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index 1832383..c8b8296 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -5,6 +5,7 @@ import sqlalchemy as sa from oa_configurator import ResolvedCDMDatabase, ResolvedConnection +from oa_configurator import qualified, schema_of, Dialect from oa_configurator.testing import DIALECT_PARAMS, isolated_test_schema from omop_alchemy.backends.sqlite import SQLiteBackend from omop_alchemy.cdm.base.indexing import omop_index_name @@ -61,12 +62,22 @@ def reconcile_engine(request) -> _ReconcileEngine: """ if request.param == "postgresql": resolved = request.getfixturevalue("pg_db").resolved - engine = request.getfixturevalue("pg_session").get_bind() + engine = request.getfixturevalue("pg_schema_session").get_bind() + schema = schema_of(engine) + resolved = dataclasses.replace( + resolved, + schema_name=schema, + vocab_schema=schema, + results_schema=schema, + ) else: engine = request.getfixturevalue("fresh_engine") resolved = _sqlite_resolved(engine) create_missing_tables(engine) - manage_indexes(engine, enable=True) + if request.param == Dialect.POSTGRESQL: + manage_indexes(engine, enable=True, db_schema=schema) + else: + manage_indexes(engine, enable=True) return _ReconcileEngine(engine, resolved) @@ -107,8 +118,10 @@ def test_reconcile_schema_reports_no_drift_on_fresh_database(reconcile_engine): def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(reconcile_engine): engine, resolved = reconcile_engine with engine.begin() as connection: - connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") - connection.exec_driver_sql("CREATE INDEX idx_gender ON person (gender_concept_id)") + connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX)}") + connection.exec_driver_sql( + f"CREATE INDEX idx_gender ON {qualified(connection, 'person')} (gender_concept_id)" + ) report = reconcile_schema(engine, resolved=resolved) issues = _person_gender_issues(report) @@ -123,8 +136,10 @@ def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(rec def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(reconcile_engine): engine, resolved = reconcile_engine with engine.begin() as connection: - connection.exec_driver_sql(f"DROP INDEX {PERSON_GENDER_INDEX}") - connection.exec_driver_sql("CREATE INDEX idx_gender ON person (gender_concept_id)") + connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX)}") + connection.exec_driver_sql( + f"CREATE INDEX idx_gender ON {qualified(connection, 'person')} (gender_concept_id)" + ) report = reconcile_schema(engine, resolved=resolved) person_result = next(r for r in report.table_results if r.table_name == "person") @@ -167,6 +182,7 @@ def test_reconcile_schema_reports_relocated_when_table_found_in_another_schema(p # public may also legitimately carry a person table from an # unrelated database/test, so assert schema_b is among the # relocated schemas rather than the only one reported. + assert person_issue.actual is not None assert schema_b in person_issue.actual.split(", ") assert is_blocking_issue(person_issue) From 9c37c3a4e1a6c152e023c070a40a463814aa7c1f Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Mon, 7 Sep 2026 06:00:55 +0000 Subject: [PATCH 08/32] Make test more robust by checking for sequential due to modifying the underlying DB data --- docs/getting-started/quickstart.md | 12 ++-- tests/conftest.py | 95 +++++++++++++++++------------- tests/test_indexes.py | 8 ++- tests/test_schema_reconcile.py | 4 +- 4 files changed, 67 insertions(+), 52 deletions(-) diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index 40f790f..ea1f9a7 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -23,12 +23,10 @@ See [Configuration](configuration.md) for the full field reference. ## Running PostgreSQL tests locally -The test suite includes PostgreSQL-specific tests that skip automatically unless a `test_cdm_db` database is configured in `~/.config/omop/config.toml`. Tests are marked with `@pytest.mark.requires_database("test_cdm_db")` and skipped at collection time when the database is absent — no manual filtering required. +The test suite includes PostgreSQL-specific tests that skip automatically unless a `test_cdm_db_pg` database is configured in `~/.config/omop/config.toml`. They're resolved via oa-configurator's `isolated_test_database()`, wrapped in this repo's own `pg_db`/`pg_engine`/`pg_session` fixtures, and marked `@pytest.mark.postgresql` (plus `db_dialect` where a test could corrupt shared ORM metadata if run alongside SQLite in the same process). `addopts = "-m 'not db_dialect'"` excludes those by default, so a plain `pytest` run skips them with no manual filtering required. Run them explicitly with `pytest -m postgresql`. -> **This test database is destructive.** The test suite drops and recreates the entire `public` -> schema on every run. `test_cdm_db` must point to a **dedicated, empty test database**, never -> to a database that contains real data. The test suite enforces this: it fails loudly (not skips) if the -> configured database is not marked `test_only = true` in your config. +!!! warning "This test database is destructive." + `pg_session`-backed tests drop and recreate every non-system schema (not just `public`) both before and after each test. `test_cdm_db_pg` must point to a **dedicated, empty test database**, never to a database that contains real data. The test suite enforces this: it fails loudly if the configured database is not marked `test_only = true` in your config. The suite runs sequentially by design and does not support `pytest-xdist`: it fails loudly under `-n 2` or higher rather than racing another worker's reset. **Step 1 — Register a test database connection:** @@ -36,7 +34,7 @@ The test suite includes PostgreSQL-specific tests that skip automatically unless omop-config configure omop_alchemy ``` -When prompted whether to configure a test database, answer **Y** and supply the connection details for your dedicated test PostgreSQL instance. It will be saved as `test_cdm_db` with `test_only = true`. +When prompted whether to configure a test database, answer **Y** and supply the connection details for your dedicated test PostgreSQL instance. It will be saved as `test_cdm_db_pg` with `test_only = true`. > **Note on permissions**: the test suite disables FK constraint triggers during bulk vocabulary > loads, an operation PostgreSQL restricts to superusers. Ensure the test database user has @@ -48,4 +46,4 @@ When prompted whether to configure a test database, answer **Y** and supply the pytest -v tests/ ``` -PostgreSQL tests auto-skip when `test_cdm_db` is not configured; all other tests run regardless. +PostgreSQL tests are excluded from a plain `pytest` run by default (see above); run `pytest -m postgresql` to include them, or `pytest -v tests/ -m postgresql` for verbose output. They still auto-skip if `test_cdm_db_pg` is not configured. diff --git a/tests/conftest.py b/tests/conftest.py index d31e314..abf90d3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,11 @@ import copy +import os from datetime import date from pathlib import Path import pytest import sqlalchemy as sa from orm_loader.helpers import bootstrap -from oa_configurator.testing import isolated_test_database +from oa_configurator.testing import isolated_test_database, isolated_test_schema import sqlalchemy.orm as so from sqlalchemy.orm import Session, sessionmaker @@ -371,24 +372,17 @@ def pg_db(request): @pytest.fixture def pg_engine(pg_db): - """Real, genuinely-committing PostgreSQL engine for tests that need - actual engine-building code paths (e.g. code that calls ``.connect()`` - or ``.begin()`` on what it's given, which a bare ``Connection`` can't - stand in for). - - A thin shim over ``pg_db``: reuses its already-resolved, test_only-checked - connection's own underlying ``Engine`` (``Connection.engine``) rather - than re-implementing resolution. Deliberately does not share pg_db's - rolled-back transaction: this fixture commits for real, isolated via - pg_session's own drop/recreate of the public schema instead. Function-scoped - (was session-scoped before this shim), since pg_db itself is - function-scoped and pytest fixtures can't depend on a narrower scope - than their own. + """Real, genuinely-committing PostgreSQL engine on the connection's own + default schema (``public``). ``pg_session`` resets it clean before and + after each test; this fixture just hands back the engine to run + genuine engine-building code paths against (``.connect()``/``.begin()``, + which a bare ``Connection`` can't stand in for). + + A thin shim over ``pg_db``'s own ``committing_engine``: every role + (``None``, ``"vocab"``, ``"results"``) folds back to the connection's + default, matching the single-schema setup ``pg_session`` provides. """ - return pg_db.connection.engine.execution_options( - # This fixture only ever creates/recreates the public schema below -- - # map every role back to None so vocab/results-tagged tables land - # there too, matching the single-schema setup this fixture provides. + return pg_db.committing_engine.execution_options( schema_translate_map={None: None, "vocab": None, "results": None} ) @@ -399,14 +393,22 @@ def pg_engine(pg_db): def _reset_test_database(engine: sa.Engine) -> None: """Drop every non-system schema and recreate public. - Local to this file, not a shared oa-configurator primitive: unlike - every other package's committing-engine tests (self-cleaning via - isolated_test_schema()), pg_session's literal-"public"-name dependents - (see its own docstring) need a full reset, not just one schema's worth. - Assumes this test database isn't shared with a concurrently-running - process, the same assumption pg_session's public-only reset already - made. + Only safe against a database used by one process at a time: this + suite runs sequentially by design (no ``pytest-xdist`` support), so a + single shared schema reset before/after each test is simpler than + per-test isolation and gives the same guarantee here. Fails loudly + rather than racing if that assumption is ever violated (e.g. `-n 2+` + run by mistake). """ + worker_count = os.environ.get("PYTEST_XDIST_WORKER_COUNT") + if worker_count is not None and int(worker_count) > 1: + pytest.fail( + "_reset_test_database() cannot run safely under parallel pytest-xdist " + f"workers ({worker_count} active): it drops and recreates every " + "non-system schema in a database shared across the whole test session, " + "so concurrent workers would race each other's resets. This suite is " + "sequential-only; run it without -n." + ) with engine.connect() as conn: schema_names = sa.inspect(conn).get_schema_names() for schema in schema_names: @@ -420,21 +422,13 @@ def _reset_test_database(engine: sa.Engine) -> None: @pytest.fixture def pg_session(pg_engine, cleanup_after_test): - """ - Function-scoped PostgreSQL session with a clean schema for each test. - - Resets every non-system schema (not just public) both before and after - each test, via cleanup_after_test, so a test's own committed DDL/DML - -- in public or a reserved bookkeeping schema like MAINTENANCE_SCHEMA - -- never depends on some later, unrelated test to wipe it. Cannot move - onto ``isolated_test_schema()``'s real, uniquely-named schema: some - tests request ``pg_session`` and ``pg_engine`` together and rely on - both pointing at the same physical schema (``pg_engine`` itself has no - schema of its own, only whatever the connection's own default is). - Confirmed by a real failure when this migration was tried: - ``test_load_vocab_postgres.py``'s ``pg_session, pg_engine`` tests - broke, since ``pg_engine``-only calls then targeted an unbootstrapped - schema. + """Function-scoped PostgreSQL session with a clean schema for each test. + + Resets every non-system schema (not just public) both before and + after each test, via ``cleanup_after_test``, so a test's own + committed DDL/DML -- in public or a reserved bookkeeping schema like + ``MAINTENANCE_SCHEMA`` -- never depends on some later, unrelated test + to wipe it. """ _reset_test_database(pg_engine) cleanup_after_test(lambda: _reset_test_database(pg_engine)) @@ -449,6 +443,27 @@ def pg_session(pg_engine, cleanup_after_test): session.close() +@pytest.fixture +def pg_schema_session(pg_db): + """PostgreSQL session bound to a unique committed schema. + + Use this for tests that do not assert the literal ``public`` schema. The + schema is dropped at teardown, so separate test processes cannot reset one + another's objects. + """ + with isolated_test_schema(pg_db.committing_engine, prefix="omop_alchemy") as schema: + engine = pg_db.committing_engine.execution_options( + schema_translate_map={None: schema, "vocab": schema, "results": schema} + ) + bootstrap(engine, create=True) + session = so.Session(engine, expire_on_commit=False) + try: + yield session + finally: + session.rollback() + session.close() + + @pytest.fixture(scope="function") def session(engine) -> Session: # type: ignore """ diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 35570f7..20eb88f 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -47,9 +47,11 @@ def indexed_engine(request): not SQLite-specific. Only the postgresql param ever requests pg_session, so the sqlite param never needs a database. - pg_session's own isolation only resets the public schema, never - MAINTENANCE_SCHEMA (a separate, fixed schema bookkeeping tables live - in), so the bookkeeping table is dropped here explicitly. Otherwise + pg_session's own isolation gives every test a fresh, uniquely-named + CDM schema (see pg_engine), but MAINTENANCE_SCHEMA is a separate, + fixed, shared schema bookkeeping tables always live in regardless of + that per-test uniqueness, so it is never reset by pg_session at all. + The bookkeeping table is dropped here explicitly instead. Otherwise both its rows *and its very existence* stay visible to every later test, indefinitely, including tests that specifically assert it hasn't been created yet. diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index c8b8296..69d31bf 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -57,8 +57,8 @@ def reconcile_engine(request) -> _ReconcileEngine: Notes ----- DIALECT_PARAMS marks each param directly, since the postgresql param's - dynamic request.getfixturevalue("pg_session") call is invisible to - pytest's usual fixturenames-based auto-detection. + dynamic request.getfixturevalue("pg_schema_session") call is invisible + to pytest's usual fixturenames-based auto-detection. """ if request.param == "postgresql": resolved = request.getfixturevalue("pg_db").resolved From 1c7d0f6282e3b75471cdfd0b943b82c285ec502a Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 10 Sep 2026 05:54:11 +0000 Subject: [PATCH 09/32] Common use-case docs --- docs/getting-started/common-use-cases.md | 128 +++++++++++++++++++++++ docs/getting-started/configuration.md | 1 + mkdocs.yml | 1 + 3 files changed, 130 insertions(+) create mode 100644 docs/getting-started/common-use-cases.md diff --git a/docs/getting-started/common-use-cases.md b/docs/getting-started/common-use-cases.md new file mode 100644 index 0000000..a98f281 --- /dev/null +++ b/docs/getting-started/common-use-cases.md @@ -0,0 +1,128 @@ +# Common Use Cases + +This pages details common use-cases and setups for users and how to wrap their established OMOP CDM with `omop-alchemy` and configure it with `oa-configurator`. + +!!! note "Important References" + - [**`oa-configurator` config reference**](https://AustralianCancerDataNetwork.github.io/oa-configurator/config-reference/): Information about the config file created and stored by default at `~/.config/omop/config.toml` + - [**`omop-alchemy` and relationship to OMOP CDM**](): Important how schemas capture specific tables. + - [**`oa_configurator`'s architecture guide`**](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/): Information about the core concepts, supported configuration templates, schema translation and provenance guard, and more. + - [**`omop-alchemy`'s maintenance module**](maintenance.md): full command reference + + +--- + +## Vocabulary tables in a separate schema, same server + +!!! note "Scenario" + - Your `Concept` table (and the rest of the vocabulary) lives in a `myvocab` schema + - Vocabulary is separate from your clinical tables' schema. + - [Reading how `omop-alchemy` bundles tables in schemas]() reveals, that the `vocab_schema` configuration key is responsible for the `Concept` table + +### Solution +Set `vocab_schema` to `myvocab` during interactive configuration +```bash +omop-config configure omop_alchemy +``` + +The resulting entry in `config.toml` will look like this: + +```toml +[connections.] +dialect = ... +host = ... +.... + +[databases.] +kind = "cdm" +connection = "" +schema_name = "" +vocab_schema = "myvocab" # <- overwritten schema map + +[tools.omop_alchemy] +cdm_db = "" +``` + +Every vocabulary-tagged table (see [Documentation for more details]()) now resolves into `myvocab` automatically. +This does not require any model changes. `results_schema` works the same way for results tables and is also defined in the [Documentation]() + +### Troubleshooting + +#### 1. You misconfigured the schema wrong for your select database + +No issues. Just re-run the configuration command again: +```bash +omop-config configure omop_alchemy +``` + +The CLI wizard will guide you through the entire setup again. You can changed/modify settings. Previously configured fields are now the default and can just be accepted by pressing 'Enter'. + +--- + +## Vocabulary on an entirely separate server + +!!! note "Scenario" + - Your entire CDM vocabulary lives on a separate physical DB server (e.g. a shared vocabnulary instance resued across multiple CDM deployments) + - You checked the documentation for [supported dialects in `omop-alchemy`](https://AustralianCancerDataNetwork.github.io/oa-configurator/config-reference/#supported-dialects ) and confirmed that your separate DB server is supported + + +### Solution + +Configure a separate `vocab_connection` during the setup of `omop-alchemy` when prompted: +```bash +omop-config configure omop_alchemy +``` + +```toml +[connections.cdm] # <- your CDM connection +dialect = "postgresql+psycopg" +host = "cdm-db.internal" +database_name = "cdm" + +[connections.vocab] # <- your vocab connection +dialect = "postgresql+psycopg" +host = "vocab-db.internal" +database_name = "vocab" + +[databases.cdm_db] +kind = "cdm" +connection = "cdm" +vocab_connection = "vocab" # <- references your vocabulary DB +``` + +!!! warning "Queries spanning both databases" + Reads that only touch vocabulary tables route to the `vocab` connection automatically. However, a query joining a vocabulary table against a clinical table (that lives in the `cdm` connection/DB) can't be answered by one physical connection in the underlying backend `SQLAlchemy`. `omop-alchemy` resolves those by querying each side separately and then merging the results in Python. Large queries therefore require significant memory budgets depending on the query. + +--- + +## Migrating an existing deployment to a schema split + +!!! note "Scenario" + - You are moving from one schema holding everything to a real vocabulary/results splits, **or** + - You are renaming a schema on a database `omop-alchemy` has already created tables in. + - **Assumptions:** + - your database for the CDM is named `my_db` in `config.toml` + - there is an entry called `[databases.my_db]`, and + - `[tools.omop_alchemy]` lists it as `cdm_db="my_db"` + - you want to move all your tables governed by the interal `vocab` schema to schema `myvocab` + +### Solution + +[`oa-configurator`'s schema provenance guard](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/#schema-provenance-guard) records which physical schema each role last resolved to, and refuses to run `create-missing-tables` if the configured schema for a role has silently changed since the last run. This mechanism is in place to stop a misconfiguration from creating an orphaned second copy of your tables. To make a genuine change deliberately: + +1. Update `vocab_schema`/`results_schema`/`schema_name` in `config.toml` through reconfiguration + ```bash + omop-config configure omop_alchemy + ``` +2. Move your actual data to the new schema yourself using access to the database. + - This is **NEVER** done automatically to preserve data integrity from our end. +3. Record the new schema as the accepted baseline following the assumptions listed in "Scenario" above: + ```bash + omop-alchemy acknowledge-schema-migration --database -my_db --role vocab --new-schema myvocab --reason "moving vocab off the shared schema" + ``` +4. Once you've confirmed the new schema is correct, clean up the old one: + ```bash + omop-alchemy drop-orphan-schema-tables --database cdm_db --schema old_vocab_schema --confirm + ``` + Omit `--confirm` first to preview what would be dropped. + + diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 8483ced..6153493 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -82,5 +82,6 @@ See the [oa-configurator integration guide](https://AustralianCancerDataNetwork. ## Further reading +- [Common Use Cases](common-use-cases.md): worked examples for vocab/results schema splits, a separate vocabulary server, and migrating an existing deployment's schema layout - [oa_configurator quickstart](https://AustralianCancerDataNetwork.github.io/oa-configurator/quickstart/): full config reference, CLI walkthrough - [oa_configurator integration guide](https://AustralianCancerDataNetwork.github.io/oa-configurator/integration/): multi-package setups diff --git a/mkdocs.yml b/mkdocs.yml index cebc5db..0cff547 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -69,6 +69,7 @@ nav: - Installation: getting-started/installation.md - Configuration: getting-started/configuration.md - Quickstart: getting-started/quickstart.md + - Common Use Cases: getting-started/common-use-cases.md - Maintenance CLI: getting-started/maintenance.md - CLI Reference: From f7d4402606522e805c27e03ee8874be487fdff74 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 10 Sep 2026 06:36:43 +0000 Subject: [PATCH 10/32] Populate oa-config links --- docs/getting-started/common-use-cases.md | 14 +++++++------- docs/getting-started/configuration.md | 23 +++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/docs/getting-started/common-use-cases.md b/docs/getting-started/common-use-cases.md index a98f281..796aa52 100644 --- a/docs/getting-started/common-use-cases.md +++ b/docs/getting-started/common-use-cases.md @@ -4,7 +4,7 @@ This pages details common use-cases and setups for users and how to wrap their e !!! note "Important References" - [**`oa-configurator` config reference**](https://AustralianCancerDataNetwork.github.io/oa-configurator/config-reference/): Information about the config file created and stored by default at `~/.config/omop/config.toml` - - [**`omop-alchemy` and relationship to OMOP CDM**](): Important how schemas capture specific tables. + - [**`omop-alchemy` and relationship to OMOP CDM**](configuration.md#cdm-table-roles): Important how schemas capture specific tables. - [**`oa_configurator`'s architecture guide`**](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/): Information about the core concepts, supported configuration templates, schema translation and provenance guard, and more. - [**`omop-alchemy`'s maintenance module**](maintenance.md): full command reference @@ -13,10 +13,10 @@ This pages details common use-cases and setups for users and how to wrap their e ## Vocabulary tables in a separate schema, same server -!!! note "Scenario" +!!! example "Scenario" - Your `Concept` table (and the rest of the vocabulary) lives in a `myvocab` schema - Vocabulary is separate from your clinical tables' schema. - - [Reading how `omop-alchemy` bundles tables in schemas]() reveals, that the `vocab_schema` configuration key is responsible for the `Concept` table + - [Reading how `omop-alchemy` bundles tables in schemas](configuration.md#cdm-table-roles) reveals, that the `vocab_schema` configuration key is responsible for the `Concept` table ### Solution Set `vocab_schema` to `myvocab` during interactive configuration @@ -42,8 +42,8 @@ vocab_schema = "myvocab" # <- overwritten schema map cdm_db = "" ``` -Every vocabulary-tagged table (see [Documentation for more details]()) now resolves into `myvocab` automatically. -This does not require any model changes. `results_schema` works the same way for results tables and is also defined in the [Documentation]() +Every vocabulary-tagged table (see [Documentation for more details](configuration.md#cdm-table-roles)) now resolves into `myvocab` automatically. +This does not require any model changes. `results_schema` works the same way for results tables and is also defined in the [Documentation](configuration.md#cdm-table-roles) ### Troubleshooting @@ -60,7 +60,7 @@ The CLI wizard will guide you through the entire setup again. You can changed/mo ## Vocabulary on an entirely separate server -!!! note "Scenario" +!!! example "Scenario" - Your entire CDM vocabulary lives on a separate physical DB server (e.g. a shared vocabnulary instance resued across multiple CDM deployments) - You checked the documentation for [supported dialects in `omop-alchemy`](https://AustralianCancerDataNetwork.github.io/oa-configurator/config-reference/#supported-dialects ) and confirmed that your separate DB server is supported @@ -96,7 +96,7 @@ vocab_connection = "vocab" # <- references your vocabulary DB ## Migrating an existing deployment to a schema split -!!! note "Scenario" +!!! example "Scenario" - You are moving from one schema holding everything to a real vocabulary/results splits, **or** - You are renaming a schema on a database `omop-alchemy` has already created tables in. - **Assumptions:** diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 6153493..55b2f24 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -39,6 +39,29 @@ cdm_db = "cdm_db" You can also write or edit this file manually. +## CDM table roles + +OMOP_Alchemy tags every table with a logical role, matching the [OMOP CDM v5.4](https://ohdsi.github.io/CommonDataModel/cdm54.html) +categories: + +- **Clinical/derived tables**: + - `Person`, `Visit_Occurrence`, `Condition_Era`, ... + - The primary role, controlled by `schema_name` in the configuration. +- **Vocabulary Tables**: + - `Concept`, `Vocabulary`, `Concept_Relationship`, `Domain`, ... + - Controlled by `vocab_schema` in the configuration. +- **Results/analytics tables**: + - `Cohort`, `Cohort Definition`, `Condition_era`, `Drug_era`, `Dose_era`, `Observation_period` + - Controlled by `results_schema` in the configuration. + +![OMOP CDM v5.4](https://ohdsi.github.io/CommonDataModel/man/images/cdm55.png) + +Each role folds back to `schema_name` when its own field is unset, so a minimal config needs no extra fields. +Setting `vocab_schema`/`results_schema` routes just that role's tables elsewhere. See [oa_configurator's schema translate map guide](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/#schema-translate-map) for how the routing itself works, and [Common Use Cases](common-use-cases.md) for worked examples of splitting these onto different schemas or servers. + +!!! info "Misconfiguration prevention" + Misconfiguring which schema a role points at doesn't corrupt data. [`oa-configurator`'s schema provenance guard](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/#schema-provenance-guard) refuses the DDL. + ## Vocabulary loading If you plan to load OMOP vocabulary from Athena CSV files, add the path to `[tools.omop_alchemy]`: diff --git a/mkdocs.yml b/mkdocs.yml index 0cff547..3da1cbb 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -150,3 +150,4 @@ nav: - Backends: advanced/backends.md - Patient Timelines: advanced/timelines.md - PostgreSQL Full-Text Search: advanced/fulltext.md + - Vocabulary Load Performance: advanced/vocabulary_load_performance.md From 2e502c18eaf9d349cc758a600bea8c92b077a2ac Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Mon, 14 Sep 2026 02:06:21 +0000 Subject: [PATCH 11/32] Correctly tag all tables mimicking OMOP CDM --- omop_alchemy/cdm/base/__init__.py | 39 ++++++++++-- omop_alchemy/cdm/base/column_helpers.py | 19 ++++++ omop_alchemy/cdm/base/column_mixins.py | 8 +-- omop_alchemy/cdm/model/__init__.py | 59 ++++++++++++++++--- omop_alchemy/cdm/model/clinical/__init__.py | 12 +++- .../model/clinical/condition_occurrence.py | 3 + omop_alchemy/cdm/model/clinical/death.py | 7 ++- .../cdm/model/clinical/device_exposure.py | 2 + .../cdm/model/clinical/drug_exposure.py | 4 ++ .../cdm/model/clinical/measurement.py | 9 +-- .../cdm/model/clinical/observation.py | 9 +-- .../observation_period.py | 5 +- omop_alchemy/cdm/model/clinical/person.py | 13 ++-- .../model/clinical/procedure_occurrence.py | 4 ++ omop_alchemy/cdm/model/clinical/specimen.py | 5 +- omop_alchemy/cdm/model/derived/__init__.py | 10 ++-- .../cdm/model/derived/condition_era.py | 5 +- omop_alchemy/cdm/model/derived/dose_era.py | 5 +- omop_alchemy/cdm/model/derived/drug_era.py | 5 +- .../cdm/model/health_economic/cost.py | 2 + .../health_economic/payer_plan_period.py | 5 +- .../cdm/model/health_system/care_site.py | 5 +- .../cdm/model/health_system/location.py | 2 + .../cdm/model/health_system/provider.py | 5 +- .../cdm/model/health_system/visit_detail.py | 15 +++-- .../model/health_system/visit_occurrence.py | 20 ++++--- omop_alchemy/cdm/model/metadata/cdm_source.py | 1 + omop_alchemy/cdm/model/metadata/metadata.py | 2 + omop_alchemy/cdm/model/structural/episode.py | 7 ++- .../cdm/model/structural/episode_event.py | 5 +- .../cdm/model/structural/fact_relationship.py | 1 + omop_alchemy/cdm/model/unstructured/note.py | 4 ++ .../cdm/model/unstructured/note_nlp.py | 5 +- omop_alchemy/maintenance/tables.py | 11 ++++ .../oncology/oncology_drug_exposure.py | 7 +++ .../oncology/oncology_procedure_occurrence.py | 7 +++ .../episodes/handling/resolved_event.py | 7 +++ tests/conftest.py | 12 ++-- ...st_backends_non_default_schema_postgres.py | 3 +- tests/test_episodes_basic.py | 2 + tests/test_fulltext.py | 7 ++- tests/test_indexes.py | 2 +- tests/test_load_vocab_postgres.py | 3 +- tests/test_schema_provenance_guard.py | 6 +- tests/test_schema_reconcile.py | 6 +- tests/test_schema_rectify_cli.py | 2 +- ...test_vocab_results_role_wiring_postgres.py | 5 +- 47 files changed, 294 insertions(+), 88 deletions(-) rename omop_alchemy/cdm/model/{derived => clinical}/observation_period.py (83%) diff --git a/omop_alchemy/cdm/base/__init__.py b/omop_alchemy/cdm/base/__init__.py index 7c714df..9be3ab9 100644 --- a/omop_alchemy/cdm/base/__init__.py +++ b/omop_alchemy/cdm/base/__init__.py @@ -1,12 +1,40 @@ from .cdm_table_base import CDMTableBase from .decorators import cdm_table, MODEL_MODULE_PREFIX -from .column_helpers import required_concept_fk, optional_concept_fk, role_fk, optional_int, required_int -from .column_mixins import ValueMixin, ReferenceTable, DatedEvent, PersonScoped, HealthSystemContext, FactTable -from .indexing import merge_table_args, omop_index, omop_primary_key_index_name, omop_table_options -from .domain_validation import DomainValidationMixin, DomainRule, ExpectedDomain +from .column_helpers import ( + required_concept_fk, + optional_concept_fk, + role_fk, + role_table, + optional_int, + required_int +) +from .column_mixins import ( + ValueMixin, + ReferenceTable, + DatedEvent, + PersonScoped, + HealthSystemContext, + FactTable +) +from .indexing import ( + merge_table_args, + omop_index, + omop_primary_key_index_name, + omop_table_options +) +from .domain_validation import ( + DomainValidationMixin, + DomainRule, + ExpectedDomain +) from .concept_validation import ConceptValidationMixin from .reference_context import ReferenceContext -from .typing import HasConceptId, HasEpisodeId, HasPersonId, DomainSemanticTable +from .typing import ( + HasConceptId, + HasEpisodeId, + HasPersonId, + DomainSemanticTable +) from .modifier_interface import ModifierTargetMixin from .cdm_constants import ModifierFieldConcepts @@ -18,6 +46,7 @@ "required_concept_fk", "optional_concept_fk", "role_fk", + "role_table", "optional_int", "required_int", "ValueMixin", diff --git a/omop_alchemy/cdm/base/column_helpers.py b/omop_alchemy/cdm/base/column_helpers.py index 9409137..e01689e 100644 --- a/omop_alchemy/cdm/base/column_helpers.py +++ b/omop_alchemy/cdm/base/column_helpers.py @@ -28,6 +28,25 @@ def role_fk(role: Role, target: str) -> str: return f"{role.value}.{target}" +def role_table(role: Role, table: str) -> str: + """Schema-qualify a bare table-name string, e.g. for ``relationship(secondary=...)``. + If a target table is schema-qualified, its metadata key is ``role.table``. + + Parameters + ---------- + role : Role + Schema role the target table is tagged with. + table : str + Unqualified table name. + + Returns + ------- + str + Schema-qualified table-reference string. + """ + return f"{role.value}.{table}" + + def required_concept_fk(): """ *required_concept_fk* diff --git a/omop_alchemy/cdm/base/column_mixins.py b/omop_alchemy/cdm/base/column_mixins.py index 6528e86..6ca2f82 100644 --- a/omop_alchemy/cdm/base/column_mixins.py +++ b/omop_alchemy/cdm/base/column_mixins.py @@ -26,7 +26,7 @@ class PersonScoped: Encodes the standard `person_id` foreign key and indexing pattern. """ - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) class ConceptTyped: """ @@ -90,9 +90,9 @@ class HealthSystemContext: Used across many clinical event tables to provide consistent join points into the health system structure. """ - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id"), nullable=True) - visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_occurrence.visit_occurrence_id"), nullable=True) - visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id"), nullable=True) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id")), nullable=True) + visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id")), nullable=True) + visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id")), nullable=True) class FactTable: """ diff --git a/omop_alchemy/cdm/model/__init__.py b/omop_alchemy/cdm/model/__init__.py index fcd394f..6006d78 100644 --- a/omop_alchemy/cdm/model/__init__.py +++ b/omop_alchemy/cdm/model/__init__.py @@ -1,12 +1,55 @@ # we have to import at least one model from each module to ensure they are registered -from .clinical import Person, Condition_Occurrence, Death, Device_Exposure, Drug_Exposure, Measurement, Observation, Procedure_Occurrence -from .derived import Observation_Period, Condition_Era, Drug_Era, Dose_Era, Cohort_Definition, Cohort -from .vocabulary import Concept, Concept_Ancestor, Domain, Vocabulary, Concept_Class, Relationship, Concept_Relationship -from .health_system import Visit_Occurrence, Care_Site, Location, Provider, Visit_Detail -from .health_economic import Cost, Payer_Plan_Period -from .structural import Episode, Episode_Event, Fact_Relationship -from .unstructured import Note, Note_NLP -from .metadata import CDM_Source, Metadata +from .clinical import ( + Person, + Condition_Occurrence, + Death, + Device_Exposure, + Drug_Exposure, + Measurement, + Observation, + Observation_Period, + Procedure_Occurrence +) +from .derived import ( + Condition_Era, + Drug_Era, + Dose_Era, + Cohort_Definition, + Cohort +) +from .vocabulary import ( + Concept, + Concept_Ancestor, + Domain, + Vocabulary, + Concept_Class, + Relationship, + Concept_Relationship +) +from .health_system import ( + Visit_Occurrence, + Care_Site, + Location, + Provider, + Visit_Detail +) +from .health_economic import ( + Cost, + Payer_Plan_Period +) +from .structural import ( + Episode, + Episode_Event, + Fact_Relationship +) +from .unstructured import ( + Note, + Note_NLP +) +from .metadata import ( + CDM_Source, + Metadata +) from .flags import ( InvalidReasonMixin, InvalidReasonFlag, diff --git a/omop_alchemy/cdm/model/clinical/__init__.py b/omop_alchemy/cdm/model/clinical/__init__.py index 9fe1046..b7e28fb 100644 --- a/omop_alchemy/cdm/model/clinical/__init__.py +++ b/omop_alchemy/cdm/model/clinical/__init__.py @@ -1,6 +1,11 @@ -from .condition_occurrence import Condition_Occurrence, Condition_OccurrenceContext, Condition_OccurrenceView +from .condition_occurrence import ( + Condition_Occurrence, + Condition_OccurrenceContext, + Condition_OccurrenceView +) from .measurement import Measurement from .observation import Observation +from .observation_period import Observation_Period from .person import Person, PersonView from .drug_exposure import Drug_Exposure from .procedure_occurrence import Procedure_Occurrence @@ -9,11 +14,12 @@ from .specimen import Specimen __all__ = [ - "Condition_Occurrence", - "Condition_OccurrenceContext", + "Condition_Occurrence", + "Condition_OccurrenceContext", "Condition_OccurrenceView", "Measurement", "Observation", + "Observation_Period", "Person", "Drug_Exposure", "Procedure_Occurrence", diff --git a/omop_alchemy/cdm/model/clinical/condition_occurrence.py b/omop_alchemy/cdm/model/clinical/condition_occurrence.py index b3bff14..a614fb4 100644 --- a/omop_alchemy/cdm/model/clinical/condition_occurrence.py +++ b/omop_alchemy/cdm/model/clinical/condition_occurrence.py @@ -34,6 +34,7 @@ class Condition_Occurrence( ): __tablename__ = "condition_occurrence" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "condition_concept_id"), omop_index(__tablename__, "visit_occurrence_id") @@ -73,6 +74,8 @@ class Condition_OccurrenceView( ModifierTargetMixin ): __tablename__ = "condition_occurrence" + # Must match Condition_Occurrence's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "condition_occurrence_id" __concept_id_col__ = "condition_concept_id" diff --git a/omop_alchemy/cdm/model/clinical/death.py b/omop_alchemy/cdm/model/clinical/death.py index 85daeed..460afb0 100644 --- a/omop_alchemy/cdm/model/clinical/death.py +++ b/omop_alchemy/cdm/model/clinical/death.py @@ -1,10 +1,12 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, CDMTableBase, cdm_table, optional_concept_fk, @@ -23,9 +25,10 @@ class Death(CDMTableBase, Base): __tablename__ = "death" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_table_options(cluster_on=omop_primary_key_index_name("death")), ) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), primary_key=True) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), primary_key=True) death_date: so.Mapped[date] = so.mapped_column(nullable=False) death_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) death_type_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() @@ -41,6 +44,8 @@ class DeathContext(ReferenceContext): class DeathView(Death, DeathContext, DomainValidationMixin): __tablename__ = "death" + # Must match Death's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { diff --git a/omop_alchemy/cdm/model/clinical/device_exposure.py b/omop_alchemy/cdm/model/clinical/device_exposure.py index c4862b2..90152af 100644 --- a/omop_alchemy/cdm/model/clinical/device_exposure.py +++ b/omop_alchemy/cdm/model/clinical/device_exposure.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date, datetime @@ -29,6 +30,7 @@ class Device_Exposure( ): __tablename__ = "device_exposure" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "device_concept_id"), omop_index(__tablename__, "visit_occurrence_id"), diff --git a/omop_alchemy/cdm/model/clinical/drug_exposure.py b/omop_alchemy/cdm/model/clinical/drug_exposure.py index ea2d9aa..a24a0e9 100644 --- a/omop_alchemy/cdm/model/clinical/drug_exposure.py +++ b/omop_alchemy/cdm/model/clinical/drug_exposure.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date, datetime from orm_loader.helpers import Base @@ -33,6 +34,7 @@ class Drug_Exposure( ): __tablename__ = "drug_exposure" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), omop_index(__tablename__, "visit_occurrence_id") @@ -77,6 +79,8 @@ class Drug_ExposureView( ): __tablename__ = "drug_exposure" + # Must match Drug_Exposure's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "drug_exposure_id" diff --git a/omop_alchemy/cdm/model/clinical/measurement.py b/omop_alchemy/cdm/model/clinical/measurement.py index 159d71f..4db3c49 100644 --- a/omop_alchemy/cdm/model/clinical/measurement.py +++ b/omop_alchemy/cdm/model/clinical/measurement.py @@ -21,6 +21,7 @@ class Measurement(Base, CDMTableBase, ValueMixin): __tablename__ = "measurement" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "measurement_concept_id"), omop_index(__tablename__, "visit_occurrence_id"), @@ -29,7 +30,7 @@ class Measurement(Base, CDMTableBase, ValueMixin): ) measurement_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) measurement_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) measurement_date: so.Mapped[date] = so.mapped_column(nullable=False) measurement_datetime: so.Mapped[Optional[datetime]] @@ -41,9 +42,9 @@ class Measurement(Base, CDMTableBase, ValueMixin): range_low: so.Mapped[Optional[float]] range_high: so.Mapped[Optional[float]] - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id")) - visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_occurrence.visit_occurrence_id")) - visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id")) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id"))) + visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id"))) + visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id"))) measurement_source_value: so.Mapped[Optional[str]] measurement_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() diff --git a/omop_alchemy/cdm/model/clinical/observation.py b/omop_alchemy/cdm/model/clinical/observation.py index 6795f48..3f6b8f3 100644 --- a/omop_alchemy/cdm/model/clinical/observation.py +++ b/omop_alchemy/cdm/model/clinical/observation.py @@ -21,6 +21,7 @@ class Observation(Base, CDMTableBase, ValueMixin): __tablename__ = "observation" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "observation_concept_id"), omop_index(__tablename__, "visit_occurrence_id"), @@ -28,7 +29,7 @@ class Observation(Base, CDMTableBase, ValueMixin): ) observation_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) observation_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")), nullable=False) observation_date: so.Mapped[date] = so.mapped_column(nullable=False) observation_datetime: so.Mapped[Optional[datetime]] @@ -38,9 +39,9 @@ class Observation(Base, CDMTableBase, ValueMixin): #value_as_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() qualifier_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() unit_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id")) - visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_occurrence.visit_occurrence_id")) - visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id")) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id"))) + visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id"))) + visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id"))) observation_source_value: so.Mapped[Optional[str]] observation_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() unit_source_value: so.Mapped[Optional[str]] diff --git a/omop_alchemy/cdm/model/derived/observation_period.py b/omop_alchemy/cdm/model/clinical/observation_period.py similarity index 83% rename from omop_alchemy/cdm/model/derived/observation_period.py rename to omop_alchemy/cdm/model/clinical/observation_period.py index e4914d1..adac764 100644 --- a/omop_alchemy/cdm/model/derived/observation_period.py +++ b/omop_alchemy/cdm/model/clinical/observation_period.py @@ -4,6 +4,7 @@ from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -17,11 +18,11 @@ class Observation_Period(CDMTableBase, Base): __table_args__ = merge_table_args( omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "period_type_concept_id"), - {"schema": Role.RESULTS.value}, + {"schema": Role.PRIMARY.value}, ) observation_period_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) observation_period_start_date: so.Mapped[date] = so.mapped_column(nullable=False) observation_period_end_date: so.Mapped[date] = so.mapped_column(nullable=False) period_type_concept_id: so.Mapped[int] = required_concept_fk() diff --git a/omop_alchemy/cdm/model/clinical/person.py b/omop_alchemy/cdm/model/clinical/person.py index a22b05b..f64b3fe 100644 --- a/omop_alchemy/cdm/model/clinical/person.py +++ b/omop_alchemy/cdm/model/clinical/person.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from sqlalchemy.ext.declarative import declared_attr from typing import Optional from datetime import date @@ -9,6 +10,7 @@ from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -28,12 +30,13 @@ from ..vocabulary import Concept from ..health_system import Location, Provider, Care_Site from .death import Death -from ..derived import Observation_Period +from .observation_period import Observation_Period @cdm_table class Person(CDMTableBase,Base,HealthSystemContext): __tablename__ = "person" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "gender_concept_id"), omop_table_options(cluster_on=omop_primary_key_index_name("person")), ) @@ -52,9 +55,9 @@ class Person(CDMTableBase,Base,HealthSystemContext): race_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() ethnicity_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() - location_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("location.location_id"), nullable=True) - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id"), nullable=True) - care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("care_site.care_site_id"), nullable=True) + location_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "location.location_id")), nullable=True) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id")), nullable=True) + care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "care_site.care_site_id")), nullable=True) person_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) gender_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) @@ -105,6 +108,8 @@ class PersonView(Person, PersonContext, DomainValidationMixin): Avoid in ETL loops. """ __tablename__ = "person" + # Must match Person's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { "gender_concept_id": ExpectedDomain("Gender"), diff --git a/omop_alchemy/cdm/model/clinical/procedure_occurrence.py b/omop_alchemy/cdm/model/clinical/procedure_occurrence.py index 4dfb65b..de803ba 100644 --- a/omop_alchemy/cdm/model/clinical/procedure_occurrence.py +++ b/omop_alchemy/cdm/model/clinical/procedure_occurrence.py @@ -1,6 +1,7 @@ from __future__ import annotations import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date from orm_loader.helpers import Base @@ -29,6 +30,7 @@ class Procedure_Occurrence(CDMTableBase, Base, PersonScoped, HealthSystemContext): __tablename__ = "procedure_occurrence" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "procedure_concept_id"), omop_index(__tablename__, "visit_occurrence_id") @@ -116,6 +118,8 @@ class Procedure_OccurrenceView( ModifierTargetMixin, ): __tablename__ = "procedure_occurrence" + # Must match Procedure_Occurrence's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __event_id_col__ = "procedure_occurrence_id" diff --git a/omop_alchemy/cdm/model/clinical/specimen.py b/omop_alchemy/cdm/model/clinical/specimen.py index f33a00e..9642d55 100644 --- a/omop_alchemy/cdm/model/clinical/specimen.py +++ b/omop_alchemy/cdm/model/clinical/specimen.py @@ -1,9 +1,11 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, CDMTableBase, cdm_table, required_concept_fk, @@ -16,12 +18,13 @@ class Specimen(CDMTableBase, Base): __tablename__ = "specimen" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "specimen_concept_id") ) specimen_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) specimen_concept_id: so.Mapped[int] = required_concept_fk() specimen_type_concept_id: so.Mapped[int] = required_concept_fk() diff --git a/omop_alchemy/cdm/model/derived/__init__.py b/omop_alchemy/cdm/model/derived/__init__.py index 4506634..550d714 100644 --- a/omop_alchemy/cdm/model/derived/__init__.py +++ b/omop_alchemy/cdm/model/derived/__init__.py @@ -3,13 +3,11 @@ from .condition_era import Condition_Era from .drug_era import Drug_Era from .dose_era import Dose_Era -from .observation_period import Observation_Period __all__ = [ - "Cohort_Definition", - "Cohort", - "Condition_Era", + "Cohort_Definition", + "Cohort", + "Condition_Era", "Drug_Era", - "Dose_Era", - "Observation_Period" + "Dose_Era", ] diff --git a/omop_alchemy/cdm/model/derived/condition_era.py b/omop_alchemy/cdm/model/derived/condition_era.py index e3dcdcb..e1072d5 100644 --- a/omop_alchemy/cdm/model/derived/condition_era.py +++ b/omop_alchemy/cdm/model/derived/condition_era.py @@ -5,6 +5,7 @@ from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -18,11 +19,11 @@ class Condition_Era(CDMTableBase, Base): __table_args__ = merge_table_args( omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "condition_concept_id"), - {"schema": Role.RESULTS.value}, + {"schema": Role.PRIMARY.value}, ) condition_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) condition_concept_id: so.Mapped[int] = required_concept_fk() condition_era_start_date: so.Mapped[date] = so.mapped_column(nullable=False) condition_era_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/derived/dose_era.py b/omop_alchemy/cdm/model/derived/dose_era.py index f1e91b6..2b1246a 100644 --- a/omop_alchemy/cdm/model/derived/dose_era.py +++ b/omop_alchemy/cdm/model/derived/dose_era.py @@ -4,6 +4,7 @@ from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -18,11 +19,11 @@ class Dose_Era(CDMTableBase, Base): omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), omop_index(__tablename__, "unit_concept_id"), - {"schema": Role.RESULTS.value}, + {"schema": Role.PRIMARY.value}, ) dose_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) drug_concept_id: so.Mapped[int] = required_concept_fk() unit_concept_id: so.Mapped[int] = required_concept_fk() dose_value: so.Mapped[float] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/derived/drug_era.py b/omop_alchemy/cdm/model/derived/drug_era.py index 230525e..c0beb61 100644 --- a/omop_alchemy/cdm/model/derived/drug_era.py +++ b/omop_alchemy/cdm/model/derived/drug_era.py @@ -5,6 +5,7 @@ from oa_configurator import Role from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -18,11 +19,11 @@ class Drug_Era(CDMTableBase, Base): __table_args__ = merge_table_args( omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), - {"schema": Role.RESULTS.value}, + {"schema": Role.PRIMARY.value}, ) drug_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) drug_concept_id: so.Mapped[int] = required_concept_fk() drug_era_start_date: so.Mapped[date] = so.mapped_column(nullable=False) drug_era_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/health_economic/cost.py b/omop_alchemy/cdm/model/health_economic/cost.py index 71c12a6..97c383d 100644 --- a/omop_alchemy/cdm/model/health_economic/cost.py +++ b/omop_alchemy/cdm/model/health_economic/cost.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( @@ -15,6 +16,7 @@ class Cost(CDMTableBase, Base): __tablename__ = "cost" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "cost_event_id"), omop_index(__tablename__, "cost_type_concept_id"), ) diff --git a/omop_alchemy/cdm/model/health_economic/payer_plan_period.py b/omop_alchemy/cdm/model/health_economic/payer_plan_period.py index 61bd5cb..99df58b 100644 --- a/omop_alchemy/cdm/model/health_economic/payer_plan_period.py +++ b/omop_alchemy/cdm/model/health_economic/payer_plan_period.py @@ -1,9 +1,11 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, optional_concept_fk, @@ -15,11 +17,12 @@ class Payer_Plan_Period(CDMTableBase, Base): __tablename__ = "payer_plan_period" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), ) payer_plan_period_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) payer_plan_period_start_date: so.Mapped[date] = so.mapped_column(nullable=False) payer_plan_period_end_date: so.Mapped[date] = so.mapped_column(nullable=False) diff --git a/omop_alchemy/cdm/model/health_system/care_site.py b/omop_alchemy/cdm/model/health_system/care_site.py index ba5b868..ec6dee5 100644 --- a/omop_alchemy/cdm/model/health_system/care_site.py +++ b/omop_alchemy/cdm/model/health_system/care_site.py @@ -1,9 +1,11 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, optional_concept_fk, @@ -17,6 +19,7 @@ class Care_Site(CDMTableBase, Base): __tablename__ = "care_site" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "place_of_service_concept_id"), omop_index(__tablename__, "location_id"), omop_table_options(cluster_on=omop_primary_key_index_name("care_site")), @@ -25,7 +28,7 @@ class Care_Site(CDMTableBase, Base): care_site_id: so.Mapped[int] = so.mapped_column(primary_key=True) care_site_name: so.Mapped[Optional[str]] = so.mapped_column(sa.String(255), nullable=True) place_of_service_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() - location_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("location.location_id"), nullable=True) + location_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "location.location_id")), nullable=True) care_site_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) place_of_service_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) diff --git a/omop_alchemy/cdm/model/health_system/location.py b/omop_alchemy/cdm/model/health_system/location.py index 95e8c87..66732c3 100644 --- a/omop_alchemy/cdm/model/health_system/location.py +++ b/omop_alchemy/cdm/model/health_system/location.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from orm_loader.helpers import Base @@ -17,6 +18,7 @@ class Location(CDMTableBase, Base): __tablename__ = "location" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "country_concept_id"), omop_table_options(cluster_on=omop_primary_key_index_name("location")), ) diff --git a/omop_alchemy/cdm/model/health_system/provider.py b/omop_alchemy/cdm/model/health_system/provider.py index f2c6975..4462a83 100644 --- a/omop_alchemy/cdm/model/health_system/provider.py +++ b/omop_alchemy/cdm/model/health_system/provider.py @@ -1,8 +1,10 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, optional_concept_fk, @@ -16,6 +18,7 @@ class Provider(CDMTableBase, Base): __tablename__ = "provider" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "specialty_concept_id"), omop_index(__tablename__, "care_site_id"), omop_index(__tablename__, "gender_concept_id"), @@ -27,7 +30,7 @@ class Provider(CDMTableBase, Base): npi: so.Mapped[Optional[str]] = so.mapped_column(sa.String(20), nullable=True) dea: so.Mapped[Optional[str]] = so.mapped_column(sa.String(20), nullable=True) specialty_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() - care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("care_site.care_site_id"), nullable=True) + care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "care_site.care_site_id")), nullable=True) year_of_birth: so.Mapped[Optional[int]] = so.mapped_column(sa.Integer, nullable=True) gender_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() provider_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) diff --git a/omop_alchemy/cdm/model/health_system/visit_detail.py b/omop_alchemy/cdm/model/health_system/visit_detail.py index a961e4f..342ba9b 100644 --- a/omop_alchemy/cdm/model/health_system/visit_detail.py +++ b/omop_alchemy/cdm/model/health_system/visit_detail.py @@ -1,10 +1,12 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -17,30 +19,31 @@ class Visit_Detail(CDMTableBase, Base): __tablename__ = "visit_detail" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "visit_detail_concept_id"), omop_index(__tablename__, "visit_occurrence_id") ) visit_detail_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) visit_detail_concept_id: so.Mapped[int] = required_concept_fk() visit_detail_start_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) visit_detail_start_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) visit_detail_end_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) visit_detail_end_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) visit_detail_type_concept_id: so.Mapped[int] = required_concept_fk() - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id"), nullable=True) - care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("care_site.care_site_id"), nullable=True) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id")), nullable=True) + care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "care_site.care_site_id")), nullable=True) visit_detail_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) visit_detail_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() admitted_from_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() admitted_from_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) discharged_to_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() discharged_to_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) - preceding_visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id"), nullable=True) - parent_visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_detail.visit_detail_id"), nullable=True) - visit_occurrence_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("visit_occurrence.visit_occurrence_id"), nullable=False) + preceding_visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id")), nullable=True) + parent_visit_detail_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_detail.visit_detail_id")), nullable=True) + visit_occurrence_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id")), nullable=False) def __repr__(self) -> str: return f"" diff --git a/omop_alchemy/cdm/model/health_system/visit_occurrence.py b/omop_alchemy/cdm/model/health_system/visit_occurrence.py index 4167219..8deb6ab 100644 --- a/omop_alchemy/cdm/model/health_system/visit_occurrence.py +++ b/omop_alchemy/cdm/model/health_system/visit_occurrence.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date from sqlalchemy.ext.declarative import declared_attr @@ -8,11 +9,13 @@ from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, + role_table, cdm_table, CDMTableBase, ReferenceContext, required_concept_fk, - optional_concept_fk, + optional_concept_fk, DomainValidationMixin, ExpectedDomain, merge_table_args, @@ -31,12 +34,13 @@ class Visit_Occurrence(CDMTableBase, Base): __tablename__ = "visit_occurrence" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "visit_concept_id"), ) visit_occurrence_id: so.Mapped[int] = so.mapped_column(primary_key=True) - person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("person.person_id"), nullable=False) + person_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "person.person_id")), nullable=False) visit_concept_id: so.Mapped[int] = required_concept_fk() visit_start_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) @@ -45,8 +49,8 @@ class Visit_Occurrence(CDMTableBase, Base): visit_end_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) visit_type_concept_id: so.Mapped[int] = required_concept_fk() - provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("provider.provider_id"), nullable=True) - care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("care_site.care_site_id"), nullable=True) + provider_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "provider.provider_id")), nullable=True) + care_site_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "care_site.care_site_id")), nullable=True) visit_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) visit_source_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() @@ -55,7 +59,7 @@ class Visit_Occurrence(CDMTableBase, Base): discharged_to_concept_id: so.Mapped[Optional[int]] = optional_concept_fk() discharged_to_source_value: so.Mapped[Optional[str]] = so.mapped_column(sa.String(50), nullable=True) - preceding_visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("visit_occurrence.visit_occurrence_id"), nullable=True) + preceding_visit_occurrence_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "visit_occurrence.visit_occurrence_id")), nullable=True) def __repr__(self) -> str: return f"" @@ -70,7 +74,7 @@ class VisitContext(ReferenceContext): def procedure_providers(cls) -> so.Mapped[list["Provider"]]: return so.relationship( "Provider", - secondary="procedure_occurrence", + secondary=role_table(Role.PRIMARY, "procedure_occurrence"), primaryjoin="Visit_Occurrence.visit_occurrence_id == Procedure_Occurrence.visit_occurrence_id", secondaryjoin="Provider.provider_id == Procedure_Occurrence.provider_id", viewonly=True, @@ -81,7 +85,7 @@ def procedure_providers(cls) -> so.Mapped[list["Provider"]]: def observation_providers(cls) -> so.Mapped[list["Provider"]]: return so.relationship( "Provider", - secondary="observation", + secondary=role_table(Role.PRIMARY, "observation"), primaryjoin="Visit_Occurrence.visit_occurrence_id == Observation.visit_occurrence_id", secondaryjoin="Provider.provider_id == Observation.provider_id", viewonly=True, @@ -90,6 +94,8 @@ def observation_providers(cls) -> so.Mapped[list["Provider"]]: class VisitView(Visit_Occurrence, VisitContext, DomainValidationMixin): __tablename__ = "visit_occurrence" + # Must match Visit_Occurrence's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { "visit_concept_id": ExpectedDomain("Visit"), diff --git a/omop_alchemy/cdm/model/metadata/cdm_source.py b/omop_alchemy/cdm/model/metadata/cdm_source.py index a561430..2696223 100644 --- a/omop_alchemy/cdm/model/metadata/cdm_source.py +++ b/omop_alchemy/cdm/model/metadata/cdm_source.py @@ -17,6 +17,7 @@ class CDM_Source(CDMTableBase, Base): __tablename__ = "cdm_source" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "cdm_version_concept_id") ) diff --git a/omop_alchemy/cdm/model/metadata/metadata.py b/omop_alchemy/cdm/model/metadata/metadata.py index 14e052c..42f724c 100644 --- a/omop_alchemy/cdm/model/metadata/metadata.py +++ b/omop_alchemy/cdm/model/metadata/metadata.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base @@ -17,6 +18,7 @@ class Metadata(CDMTableBase, Base, ValueMixin): __tablename__ = "metadata" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "metadata_concept_id", cluster=True), omop_index(__tablename__, "metadata_type_concept_id"), ) diff --git a/omop_alchemy/cdm/model/structural/episode.py b/omop_alchemy/cdm/model/structural/episode.py index fcab297..51595cc 100644 --- a/omop_alchemy/cdm/model/structural/episode.py +++ b/omop_alchemy/cdm/model/structural/episode.py @@ -1,10 +1,12 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from sqlalchemy.ext.declarative import declared_attr from typing import ClassVar, Optional, TYPE_CHECKING, List from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, cdm_table, CDMTableBase, required_concept_fk, @@ -28,6 +30,7 @@ class Episode(CDMTableBase, Base, PersonScoped): __tablename__ = "episode" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "episode_concept_id"), # following indices are not specified in the cdm but are likely to be useful for query performance @@ -37,7 +40,7 @@ class Episode(CDMTableBase, Base, PersonScoped): ) episode_id: so.Mapped[int] = so.mapped_column(primary_key=True) - episode_parent_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey("episode.episode_id"), nullable=True) + episode_parent_id: so.Mapped[Optional[int]] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "episode.episode_id")), nullable=True) episode_start_date: so.Mapped[date] = so.mapped_column(sa.Date, nullable=False) episode_start_datetime: so.Mapped[Optional[date]] = so.mapped_column(sa.DateTime, nullable=True) @@ -110,6 +113,8 @@ class EpisodeView(Episode, EpisodeContext, DomainValidationMixin): """ __tablename__ = "episode" + # Must match Episode's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { diff --git a/omop_alchemy/cdm/model/structural/episode_event.py b/omop_alchemy/cdm/model/structural/episode_event.py index d4719a1..3fb8a09 100644 --- a/omop_alchemy/cdm/model/structural/episode_event.py +++ b/omop_alchemy/cdm/model/structural/episode_event.py @@ -80,11 +80,12 @@ def clear_episode_event_target_class_cache() -> None: class Episode_Event(CDMTableBase, Base): __tablename__ = "episode_event" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "episode_id", cluster=True), omop_index(__tablename__, "episode_event_field_concept_id"), ) - episode_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey("episode.episode_id"),nullable=False,primary_key=True) + episode_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.PRIMARY, "episode.episode_id")),nullable=False,primary_key=True) event_id: so.Mapped[int] = so.mapped_column(nullable=False,primary_key=True) episode_event_field_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),nullable=False,primary_key=True) @@ -104,6 +105,8 @@ class Episode_EventView(Episode_Event, Episode_EventContext, DomainValidationMix """ __tablename__ = "episode_event" + # Must match Episode_Event's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { diff --git a/omop_alchemy/cdm/model/structural/fact_relationship.py b/omop_alchemy/cdm/model/structural/fact_relationship.py index a83e1d0..7b12051 100644 --- a/omop_alchemy/cdm/model/structural/fact_relationship.py +++ b/omop_alchemy/cdm/model/structural/fact_relationship.py @@ -15,6 +15,7 @@ class Fact_Relationship(CDMTableBase, Base): __tablename__ = "fact_relationship" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "domain_concept_id_1"), omop_index(__tablename__, "domain_concept_id_2"), omop_index(__tablename__, "relationship_concept_id"), diff --git a/omop_alchemy/cdm/model/unstructured/note.py b/omop_alchemy/cdm/model/unstructured/note.py index 369f97c..c71e4ea 100644 --- a/omop_alchemy/cdm/model/unstructured/note.py +++ b/omop_alchemy/cdm/model/unstructured/note.py @@ -1,5 +1,6 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional, TYPE_CHECKING from datetime import date, datetime @@ -29,6 +30,7 @@ class Note(CDMTableBase, Base, PersonScoped, HealthSystemContext): __tablename__ = "note" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "note_type_concept_id"), omop_index(__tablename__, "visit_occurrence_id"), @@ -114,6 +116,8 @@ class NoteContext(ReferenceContext): class NoteView(Note, NoteContext, DomainValidationMixin): __tablename__ = "note" + # Must match Note's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} __expected_domains__ = { diff --git a/omop_alchemy/cdm/model/unstructured/note_nlp.py b/omop_alchemy/cdm/model/unstructured/note_nlp.py index 0308728..9d92143 100644 --- a/omop_alchemy/cdm/model/unstructured/note_nlp.py +++ b/omop_alchemy/cdm/model/unstructured/note_nlp.py @@ -1,11 +1,13 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from typing import Optional from datetime import date from orm_loader.helpers import Base from omop_alchemy.cdm.base import ( + role_fk, CDMTableBase, cdm_table, optional_concept_fk, @@ -18,6 +20,7 @@ class Note_NLP(CDMTableBase, Base): __tablename__ = "note_nlp" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "note_id", cluster=True), omop_index(__tablename__, "note_nlp_concept_id"), ) @@ -25,7 +28,7 @@ class Note_NLP(CDMTableBase, Base): note_nlp_id: so.Mapped[int] = so.mapped_column(primary_key=True) note_id: so.Mapped[int] = so.mapped_column( - sa.ForeignKey("note.note_id"), + sa.ForeignKey(role_fk(Role.PRIMARY, "note.note_id")), nullable=False, ) diff --git a/omop_alchemy/maintenance/tables.py b/omop_alchemy/maintenance/tables.py index 6cb8c2b..94021f7 100644 --- a/omop_alchemy/maintenance/tables.py +++ b/omop_alchemy/maintenance/tables.py @@ -10,6 +10,17 @@ class TableCategory(StrEnum): """An OMOP CDM table's structural category, carrying its render style. + Represents a logical grouping of tables as defined by the + [OMOP CDM spec](https://ohdsi.github.io/CommonDataModel/). + The grouping is also reflected in the subpackage under + ``omop_alchemy.cdm.model``, where the same logical grouping is used + to organize the ORM classes. + + Notes + ----- + This logical grouping is independent of the physical schema in + which the table's data is stored (``oa_configurator.Role``). + Parameters ---------- code : str diff --git a/omop_alchemy/toolkit/analytics/oncology/oncology_drug_exposure.py b/omop_alchemy/toolkit/analytics/oncology/oncology_drug_exposure.py index 8a7af84..4ab81ac 100644 --- a/omop_alchemy/toolkit/analytics/oncology/oncology_drug_exposure.py +++ b/omop_alchemy/toolkit/analytics/oncology/oncology_drug_exposure.py @@ -2,6 +2,8 @@ from sqlalchemy.ext.hybrid import hybrid_property +from oa_configurator import Role + from omop_alchemy.cdm.model.clinical.drug_exposure import Drug_ExposureView from .concept_sets import SACT_DRUGS, resolve_sact_drug_concept_ids @@ -20,6 +22,11 @@ class OncologyDrugExposure(Drug_ExposureView): from one governed ``ConceptGroupSpec``, including its exclusions. """ + __tablename__ = "drug_exposure" + # Must match Drug_Exposure's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} + __mapper_args__ = {"concrete": False} + @hybrid_property def is_sact(self) -> bool: return self.drug_concept_id in resolve_sact_drug_concept_ids( diff --git a/omop_alchemy/toolkit/analytics/oncology/oncology_procedure_occurrence.py b/omop_alchemy/toolkit/analytics/oncology/oncology_procedure_occurrence.py index 72b02d8..a0ff288 100644 --- a/omop_alchemy/toolkit/analytics/oncology/oncology_procedure_occurrence.py +++ b/omop_alchemy/toolkit/analytics/oncology/oncology_procedure_occurrence.py @@ -2,6 +2,8 @@ from sqlalchemy.ext.hybrid import hybrid_property +from oa_configurator import Role + from omop_alchemy.cdm.model.clinical.procedure_occurrence import ( Procedure_OccurrenceView, ) @@ -33,6 +35,11 @@ class OncologyProcedure(Procedure_OccurrenceView): determine membership, and reporting "no" would silently misclassify. """ + __tablename__ = "procedure_occurrence" + # Must match Procedure_Occurrence's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} + __mapper_args__ = {"concrete": False} + @hybrid_property def is_radiotherapy(self) -> bool: return self.procedure_concept_id in resolve_rt_procedure_concept_ids( diff --git a/omop_alchemy/toolkit/episodes/handling/resolved_event.py b/omop_alchemy/toolkit/episodes/handling/resolved_event.py index 75133eb..69bc75a 100644 --- a/omop_alchemy/toolkit/episodes/handling/resolved_event.py +++ b/omop_alchemy/toolkit/episodes/handling/resolved_event.py @@ -6,6 +6,8 @@ import sqlalchemy.orm as so from sqlalchemy.ext.declarative import declared_attr +from oa_configurator import Role + from omop_alchemy.cdm.base import ModifierFieldConcepts from omop_alchemy.cdm.model.structural.episode_event import Episode_EventView @@ -59,6 +61,11 @@ class declares it, or the target row itself is missing. through ordinary ``episode.episode_events`` traversal. """ + __tablename__ = "episode_event" + # Must match Episode_Event's schema, or SQLAlchemy silently builds a second, unlinked Table object. + __table_args__ = {"schema": Role.PRIMARY.value} + __mapper_args__ = {"concrete": False} + @classmethod def recognized_field_concept_ids(cls) -> set[int]: return _known_modifier_field_concept_ids() diff --git a/tests/conftest.py b/tests/conftest.py index abf90d3..7b8777a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import sqlalchemy as sa from orm_loader.helpers import bootstrap from oa_configurator.testing import isolated_test_database, isolated_test_schema +from oa_configurator import SCHEMA_TRANSLATE_MAP_KEY, Role import sqlalchemy.orm as so from sqlalchemy.orm import Session, sessionmaker @@ -13,8 +14,7 @@ from omop_alchemy.config import OmopAlchemyConfig from omop_alchemy.maintenance.cli_vocab import _load_vocab_model_csv -from omop_alchemy.cdm.model.clinical import Condition_Occurrence, Person -from omop_alchemy.cdm.model.derived import Observation_Period +from omop_alchemy.cdm.model.clinical import Condition_Occurrence, Observation_Period, Person from omop_alchemy.cdm.model.structural import Episode, Episode_Event from omop_alchemy.cdm.model.vocabulary import ( Concept, @@ -39,7 +39,7 @@ def fresh_engine() -> Iterator[sa.Engine]: "test_cdm_db_sqlite", dialect="sqlite", future=True, - execution_options={"schema_translate_map": {None: None, "vocab": None, "results": None}}, + execution_options={SCHEMA_TRANSLATE_MAP_KEY: {Role.PRIMARY.value: None, "vocab": None, "results": None}}, ) as db: yield db.connection.engine @@ -333,7 +333,7 @@ def engine(tmp_path_factory: pytest.TempPathFactory) -> Iterator[sa.Engine]: # a single flat namespace: map every role back to None so the # vocab/results-tagged tables land in the same place they always # have here, unaffected by schema role tagging. - execution_options={"schema_translate_map": {None: None, "vocab": None, "results": None}}, + execution_options={SCHEMA_TRANSLATE_MAP_KEY: {Role.PRIMARY.value: None, "vocab": None, "results": None}}, ) as db: engine = db.connection.engine bootstrap(engine, create=True) @@ -383,7 +383,7 @@ def pg_engine(pg_db): default, matching the single-schema setup ``pg_session`` provides. """ return pg_db.committing_engine.execution_options( - schema_translate_map={None: None, "vocab": None, "results": None} + schema_translate_map={Role.PRIMARY.value: None, "vocab": None, "results": None} ) @@ -453,7 +453,7 @@ def pg_schema_session(pg_db): """ with isolated_test_schema(pg_db.committing_engine, prefix="omop_alchemy") as schema: engine = pg_db.committing_engine.execution_options( - schema_translate_map={None: schema, "vocab": schema, "results": schema} + schema_translate_map={Role.PRIMARY.value: schema, "vocab": schema, "results": schema} ) bootstrap(engine, create=True) session = so.Session(engine, expire_on_commit=False) diff --git a/tests/test_backends_non_default_schema_postgres.py b/tests/test_backends_non_default_schema_postgres.py index 196e4fb..6e5766c 100644 --- a/tests/test_backends_non_default_schema_postgres.py +++ b/tests/test_backends_non_default_schema_postgres.py @@ -18,6 +18,7 @@ import pytest import sqlalchemy as sa +from oa_configurator import Role from oa_configurator.testing import isolated_test_schema @@ -50,7 +51,7 @@ def scoped(pg_engine: sa.Engine) -> Iterator[_Scoped]: """ with isolated_test_schema(pg_engine, prefix="backends_non_default") as schema: engine = pg_engine.execution_options( - schema_translate_map={None: schema, "vocab": schema, "results": schema} + schema_translate_map={Role.PRIMARY.value: schema, "vocab": schema, "results": schema} ) yield _Scoped(engine=engine, schema=schema) diff --git a/tests/test_episodes_basic.py b/tests/test_episodes_basic.py index 7d3e7b9..1362cdf 100644 --- a/tests/test_episodes_basic.py +++ b/tests/test_episodes_basic.py @@ -1,3 +1,4 @@ +from oa_configurator import Role from omop_alchemy.cdm.base import ModifierFieldConcepts from omop_alchemy.cdm.model.structural import ( EpisodeView, @@ -19,6 +20,7 @@ class DiagnosticEpisode(ResolvedEpisodeEventMixin, EpisodeView): """Non-oncology episode view used to verify the generic extension point.""" __tablename__ = "episode" + __table_args__ = {"schema": Role.PRIMARY.value} __mapper_args__ = {"concrete": False} diff --git a/tests/test_fulltext.py b/tests/test_fulltext.py index 0ff2b76..9fc4c34 100644 --- a/tests/test_fulltext.py +++ b/tests/test_fulltext.py @@ -2,7 +2,8 @@ import pytest from sqlalchemy.dialects import postgresql from typer.testing import CliRunner -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, SCHEMA_TRANSLATE_MAP_KEY, StackConfig +from oa_configurator import Role as SchemaRole from omop_alchemy.backends import ( CONCEPT_NAME_TSVECTOR_COLUMN, @@ -43,7 +44,7 @@ def __init__(self, *, rowcount: int = 7, db_schema: str | None = "public"): self._db_schema = db_schema def get_execution_options(self) -> dict[str, object]: - return {"schema_translate_map": {None: self._db_schema}} + return {SCHEMA_TRANSLATE_MAP_KEY: {SchemaRole.PRIMARY.value: self._db_schema}} def exec_driver_sql( self, @@ -215,7 +216,7 @@ def test_fulltext_install_cli_passes_options(monkeypatch): dialect="postgresql+psycopg", host="localhost", database_name="db" ) }, - databases={"cdm_db": CDMDatabaseConfig(connection="db", schema_name="public")}, + databases={"cdm_db": CDMDatabaseConfig(connection="db", cdm_schema="public")}, ) monkeypatch.setattr( "omop_alchemy.config.load_stack_config", diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 20eb88f..93cab93 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -572,7 +572,7 @@ def test_resolving_cdm_database_with_maintenance_schema_name_raises(): StackConfig.for_session( connections={"c": ConnectionConfig(dialect="sqlite", database_name=":memory:")}, databases={ - "default": CDMDatabaseConfig(connection="c", schema_name=MAINTENANCE_SCHEMA) + "default": CDMDatabaseConfig(connection="c", cdm_schema=MAINTENANCE_SCHEMA) }, ) diff --git a/tests/test_load_vocab_postgres.py b/tests/test_load_vocab_postgres.py index e0e9ac3..50c0976 100644 --- a/tests/test_load_vocab_postgres.py +++ b/tests/test_load_vocab_postgres.py @@ -14,6 +14,7 @@ import pytest import sqlalchemy as sa +from oa_configurator import Role from omop_alchemy.backends.postgres import PostgresBackend from omop_alchemy.cdm.model.vocabulary import Concept from omop_alchemy.maintenance.cli_vocab import ( @@ -279,7 +280,7 @@ def test_db_schema_search_path_on_postgres(pg_engine, tmp_path): # schema as everything else, matching ResolvedCDMDatabase's own default # fallback behaviour when vocab_schema/results_schema aren't configured. scoped_engine = pg_engine.execution_options( - schema_translate_map={None: schema, "vocab": schema, "results": schema} + schema_translate_map={Role.PRIMARY.value: schema, "vocab": schema, "results": schema} ) try: diff --git a/tests/test_schema_provenance_guard.py b/tests/test_schema_provenance_guard.py index c7e3e34..e2f9688 100644 --- a/tests/test_schema_provenance_guard.py +++ b/tests/test_schema_provenance_guard.py @@ -22,7 +22,7 @@ import pytest import sqlalchemy as sa -from oa_configurator import SchemaDriftError +from oa_configurator import SchemaDriftError, Role from oa_configurator.domains.resources.sql import SCHEMA_PROVENANCE_SCHEMA, _schema_provenance_table from oa_configurator.testing import delete_rows_on_cleanup, isolated_test_schema @@ -58,7 +58,7 @@ def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_db, pg_engi isolated_test_schema(pg_engine, prefix="guard_wiring_b") as schema_b, ): engine_a = pg_engine.execution_options( - schema_translate_map={None: schema_a, "vocab": schema_a, "results": schema_a} + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} ) create_missing_tables( engine_a, @@ -67,7 +67,7 @@ def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_db, pg_engi ) engine_b = pg_engine.execution_options( - schema_translate_map={None: schema_b, "vocab": schema_b, "results": schema_b} + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} ) with pytest.raises(SchemaDriftError): create_missing_tables( diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index 69d31bf..9f63084 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -4,7 +4,7 @@ import pytest import sqlalchemy as sa -from oa_configurator import ResolvedCDMDatabase, ResolvedConnection +from oa_configurator import ResolvedCDMDatabase, ResolvedConnection, Role from oa_configurator import qualified, schema_of, Dialect from oa_configurator.testing import DIALECT_PARAMS, isolated_test_schema from omop_alchemy.backends.sqlite import SQLiteBackend @@ -162,7 +162,7 @@ def test_reconcile_schema_reports_relocated_when_table_found_in_another_schema(p pg_db.resolved, schema_name=schema_a, vocab_schema=schema_a, results_schema=schema_a ) engine = pg_engine.execution_options( - schema_translate_map={None: schema_a, "vocab": schema_a, "results": schema_a} + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} ) # vocabulary_included defaults to True: person's gender_concept_id FK # targets a vocab table, so excluding vocab here would leave that FK @@ -208,7 +208,7 @@ def test_reconcile_schema_with_resolved_qualifies_each_table_to_its_own_role_sch ) engine = pg_engine.execution_options( schema_translate_map={ - None: primary_schema, "vocab": vocab_schema, "results": results_schema + Role.PRIMARY.value: primary_schema, "vocab": vocab_schema, "results": results_schema } ) create_missing_tables(engine, db_schema=primary_schema, resolved=resolved) diff --git a/tests/test_schema_rectify_cli.py b/tests/test_schema_rectify_cli.py index 2244cd0..5ed6b23 100644 --- a/tests/test_schema_rectify_cli.py +++ b/tests/test_schema_rectify_cli.py @@ -46,7 +46,7 @@ def cli_stack(pg_engine, monkeypatch): ) }, databases={ - "cli_rectify_db": CDMDatabaseConfig(connection="cli_rectify_conn", schema_name="public"), + "cli_rectify_db": CDMDatabaseConfig(connection="cli_rectify_conn", cdm_schema="public"), }, ) monkeypatch.setattr("omop_alchemy.maintenance.cli_schema.load_stack_config", lambda: stack) diff --git a/tests/test_vocab_results_role_wiring_postgres.py b/tests/test_vocab_results_role_wiring_postgres.py index 6ef7e46..1392862 100644 --- a/tests/test_vocab_results_role_wiring_postgres.py +++ b/tests/test_vocab_results_role_wiring_postgres.py @@ -23,6 +23,7 @@ import sqlalchemy as sa import sqlalchemy.orm as so +from oa_configurator import Role from oa_configurator.testing import isolated_test_schema from omop_alchemy.cdm.model.clinical import Observation, Person @@ -52,7 +53,7 @@ def three_schema(pg_engine: sa.Engine) -> Iterator[_ThreeSchema]: engine = pg_engine.execution_options( schema_translate_map={ - None: clinical_schema, + Role.PRIMARY.value: clinical_schema, "vocab": vocab_schema, "results": results_schema, } @@ -124,7 +125,7 @@ def test_tables_land_in_the_schema_their_role_declares(three_schema: _ThreeSchem assert inspector.has_table("concept", schema=three_schema.vocab_schema) assert inspector.has_table("domain", schema=three_schema.vocab_schema) assert inspector.has_table("cohort", schema=three_schema.results_schema) - assert inspector.has_table("observation_period", schema=three_schema.results_schema) + assert inspector.has_table("observation_period", schema=three_schema.clinical_schema) # And not duplicated into the wrong schema. assert not inspector.has_table("concept", schema=three_schema.clinical_schema) From 9226d5f687a82589eabe61d12284e19f7382de2c Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Mon, 14 Sep 2026 02:06:47 +0000 Subject: [PATCH 12/32] Fix rich CLI errors in CI --- tests/conftest.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index 7b8777a..879c438 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ from pathlib import Path import pytest import sqlalchemy as sa +import typer.rich_utils as _typer_rich_utils from orm_loader.helpers import bootstrap from oa_configurator.testing import isolated_test_database, isolated_test_schema from oa_configurator import SCHEMA_TRANSLATE_MAP_KEY, Role @@ -26,6 +27,14 @@ Vocabulary, ) +# typer forces colorized rich error/output rendering when GITHUB_ACTIONS (or +# FORCE_COLOR / PY_COLORS) is set -- see typer.rich_utils.FORCE_TERMINAL. Under +# GitHub Actions that injects ANSI escapes into CLI output, breaking tests that +# assert on plain-substring message content (e.g. "no such option: --foo"). The +# force feeds every typer rich Console, so clear it here: tests then see the same +# uncolored output everywhere; real users still get colour in a real terminal. +_typer_rich_utils.FORCE_TERMINAL = None + @pytest.fixture def fresh_engine() -> Iterator[sa.Engine]: From 903ee50bf94d8a36afc128898a06fdde3aa7ea12 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Mon, 14 Sep 2026 03:30:03 +0000 Subject: [PATCH 13/32] Small docs update --- docs/getting-started/common-use-cases.md | 4 ++-- docs/getting-started/configuration.md | 22 +++++++++++----------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/getting-started/common-use-cases.md b/docs/getting-started/common-use-cases.md index 796aa52..31fc9fe 100644 --- a/docs/getting-started/common-use-cases.md +++ b/docs/getting-started/common-use-cases.md @@ -35,7 +35,7 @@ host = ... [databases.] kind = "cdm" connection = "" -schema_name = "" +cdm_schema = "" vocab_schema = "myvocab" # <- overwritten schema map [tools.omop_alchemy] @@ -109,7 +109,7 @@ vocab_connection = "vocab" # <- references your vocabulary DB [`oa-configurator`'s schema provenance guard](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/#schema-provenance-guard) records which physical schema each role last resolved to, and refuses to run `create-missing-tables` if the configured schema for a role has silently changed since the last run. This mechanism is in place to stop a misconfiguration from creating an orphaned second copy of your tables. To make a genuine change deliberately: -1. Update `vocab_schema`/`results_schema`/`schema_name` in `config.toml` through reconfiguration +1. Update `vocab_schema`/`results_schema`/`cdm_schema` in `config.toml` through reconfiguration ```bash omop-config configure omop_alchemy ``` diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 55b2f24..ce6c026 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -29,9 +29,9 @@ password = "changeme" database_name = "omop_cdm" [databases.cdm_db] -kind = "cdm" -connection = "cdm" -schema_name = "omop" +kind = "cdm" +connection = "cdm" +cdm_schema = "omop" [tools.omop_alchemy] cdm_db = "cdm_db" @@ -44,19 +44,19 @@ You can also write or edit this file manually. OMOP_Alchemy tags every table with a logical role, matching the [OMOP CDM v5.4](https://ohdsi.github.io/CommonDataModel/cdm54.html) categories: -- **Clinical/derived tables**: - - `Person`, `Visit_Occurrence`, `Condition_Era`, ... - - The primary role, controlled by `schema_name` in the configuration. -- **Vocabulary Tables**: - - `Concept`, `Vocabulary`, `Concept_Relationship`, `Domain`, ... +- **Clinical/derived tables** (`Role.PRIMARY`): + - All other tables not captured by the configurations below + - Controlled by `cdm_schema` in the configuration. +- **Vocabulary tables** (`Role.VOCAB`): + - `concept`, `concept_ancestor`, `concept_class`, `concept_relationship`, `concept_synonym`, `domain`, `drug_strength`, `relationship`, `source_to_concept_map`, `vocabulary` - Controlled by `vocab_schema` in the configuration. -- **Results/analytics tables**: - - `Cohort`, `Cohort Definition`, `Condition_era`, `Drug_era`, `Dose_era`, `Observation_period` +- **Results/analytics tables** (`Role.RESULTS`): + - `cohort`, `cohort_Definition` - Controlled by `results_schema` in the configuration. ![OMOP CDM v5.4](https://ohdsi.github.io/CommonDataModel/man/images/cdm55.png) -Each role folds back to `schema_name` when its own field is unset, so a minimal config needs no extra fields. +Each role folds back to `cdm_schema` when its own field is unset, so a minimal config needs no extra fields. Setting `vocab_schema`/`results_schema` routes just that role's tables elsewhere. See [oa_configurator's schema translate map guide](https://AustralianCancerDataNetwork.github.io/oa-configurator/architecture/#schema-translate-map) for how the routing itself works, and [Common Use Cases](common-use-cases.md) for worked examples of splitting these onto different schemas or servers. !!! info "Misconfiguration prevention" From 964fb7a3f7c4156181d8d2e94a94c707cbf3d8fc Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 00:30:31 +0000 Subject: [PATCH 14/32] Remove schema provenance CLI after moving it to oa-configurator --- docs/getting-started/common-use-cases.md | 6 +- .../maintenance/cli_schema_rectify.py | 98 ------------ tests/test_schema_rectify_cli.py | 149 ------------------ 3 files changed, 4 insertions(+), 249 deletions(-) delete mode 100644 omop_alchemy/maintenance/cli_schema_rectify.py delete mode 100644 tests/test_schema_rectify_cli.py diff --git a/docs/getting-started/common-use-cases.md b/docs/getting-started/common-use-cases.md index 31fc9fe..ba14b08 100644 --- a/docs/getting-started/common-use-cases.md +++ b/docs/getting-started/common-use-cases.md @@ -117,12 +117,14 @@ vocab_connection = "vocab" # <- references your vocabulary DB - This is **NEVER** done automatically to preserve data integrity from our end. 3. Record the new schema as the accepted baseline following the assumptions listed in "Scenario" above: ```bash - omop-alchemy acknowledge-schema-migration --database -my_db --role vocab --new-schema myvocab --reason "moving vocab off the shared schema" + omop-config acknowledge-schema-migration --database my_db --role vocab --new-schema myvocab --reason "moving vocab off the shared schema" ``` 4. Once you've confirmed the new schema is correct, clean up the old one: ```bash - omop-alchemy drop-orphan-schema-tables --database cdm_db --schema old_vocab_schema --confirm + omop-config drop-orphan-schema-tables --database cdm_db --schema old_vocab_schema --confirm ``` Omit `--confirm` first to preview what would be dropped. + Both commands live in `oa-configurator`, not `omop-alchemy` as they're generic over any `[databases.*]` entry, not CDM-specific. + diff --git a/omop_alchemy/maintenance/cli_schema_rectify.py b/omop_alchemy/maintenance/cli_schema_rectify.py deleted file mode 100644 index 5f32bbf..0000000 --- a/omop_alchemy/maintenance/cli_schema_rectify.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Rectify domain: acknowledge a deliberate schema change, or clean up orphaned -tables left behind by one. - -Every action requires an explicit target and confirmation; nothing here -auto-detects or auto-deletes. -""" - -from __future__ import annotations - -from dataclasses import dataclass - -import sqlalchemy as sa -from oa_configurator import ResolvedCDMDatabase, Resolver, StackConfig, schema_inspect - -from ..backends import backend_supports, resolve_backend - - -@dataclass(frozen=True) -class OrphanTablePreview: - """One table found in a candidate orphan schema, with an approximate row count.""" - - table_name: str - approximate_row_count: int | None - - -def preview_orphan_schema_tables(connection: sa.Connection, schema: str) -> list[OrphanTablePreview]: - """List tables physically present in schema, with an approximate row count - where the backend supports one (None otherwise). - """ - table_names = schema_inspect(connection, schema=schema).get_table_names() - backend = resolve_backend(connection.engine) - counts: dict[str, int] = ( - backend.approximate_row_counts(connection, schema) - if backend_supports(backend, "approximate_row_counts") - else {} - ) - return [ - OrphanTablePreview(table_name=name, approximate_row_count=counts.get(name)) - for name in table_names - ] - - -def schema_is_a_current_target(stack: StackConfig, schema: str) -> str | None: - """Return the name of a configured database whose current schema (or - vocab/results schema) equals schema, or None. - - Checks every database in the stack, not just the one named on the - command line, since two CDM databases can legitimately share one - vocabulary schema; dropping tables there would destroy a database a - different entry is actively using. - """ - resolver = Resolver(stack) - for name in stack.databases: - try: - resolved = resolver.resolve_database(name) - except Exception: - continue - candidates: set[str | None] = {resolved.schema_name} - if isinstance(resolved, ResolvedCDMDatabase): - candidates.add(resolved.vocab_schema) - candidates.add(resolved.results_schema) - if schema in candidates: - return name - return None - - -def drop_orphan_schema_tables( - connection: sa.Connection, - *, - stack: StackConfig, - orphan_schema: str, - confirm: bool, -) -> list[OrphanTablePreview]: - """Drop every table found in *orphan_schema*, after the cross-entry safety check. - - Preview-only when *confirm* is False: returns what would be dropped - without touching anything. - - Raises - ------ - RuntimeError - If *orphan_schema* is the current schema target of any configured - database/role in *stack*. - """ - blocking = schema_is_a_current_target(stack, orphan_schema) - if blocking is not None: - raise RuntimeError( - f"Refusing to drop tables in schema {orphan_schema!r}: it is the current schema " - f"target of database {blocking!r}. Reconfigure or drop that database entry first " - "if this schema is genuinely meant to be retired." - ) - preview = preview_orphan_schema_tables(connection, orphan_schema) - if confirm: - for item in preview: - connection.execute( - sa.text(f'DROP TABLE IF EXISTS "{orphan_schema}"."{item.table_name}" CASCADE') - ) - return preview diff --git a/tests/test_schema_rectify_cli.py b/tests/test_schema_rectify_cli.py deleted file mode 100644 index 5ed6b23..0000000 --- a/tests/test_schema_rectify_cli.py +++ /dev/null @@ -1,149 +0,0 @@ -"""Rectify CLI: acknowledge-schema-migration / drop-orphan-schema-tables. - -Live-Postgres regression against real StackConfig/Resolver plumbing, not -mocked, since these commands' whole point is writing/reading real database -state. Monkeypatches cli_schema.load_stack_config (the name as imported into -that module) to point at a scratch StackConfig built from pg_engine's own -connection, rather than touching the real on-disk config. -""" - -from __future__ import annotations - -import uuid - -import pytest -import sqlalchemy as sa -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig -from oa_configurator.domains.resources.sql import _provenance_schema_for, _schema_provenance_table, ensure_schema -from oa_configurator.testing import isolated_test_schema -from sqlalchemy.engine import make_url -from typer.testing import CliRunner - -from omop_alchemy.maintenance.cli import app - -pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] - -runner = CliRunner() - - -@pytest.fixture -def cli_stack(pg_engine, monkeypatch): - """A StackConfig with one real, test_only Postgres connection/database - entry, pointed at pg_engine's own live server, the same connection every - other Postgres test in this repo already uses.""" - url = make_url(pg_engine.url).render_as_string(hide_password=False) - conn_url = make_url(url) - stack = StackConfig.for_session( - connections={ - "cli_rectify_conn": ConnectionConfig( - dialect=conn_url.drivername, - host=conn_url.host, - port=conn_url.port, - user=conn_url.username, - password=conn_url.password, - database_name=conn_url.database, - test_only=True, - ) - }, - databases={ - "cli_rectify_db": CDMDatabaseConfig(connection="cli_rectify_conn", cdm_schema="public"), - }, - ) - monkeypatch.setattr("omop_alchemy.maintenance.cli_schema.load_stack_config", lambda: stack) - return stack - - -def _cleanup_provenance_row(pg_engine, *, database_name: str, role: str) -> None: - with pg_engine.begin() as conn: - ensure_schema(conn, _provenance_schema_for(conn)) - table = _schema_provenance_table(_provenance_schema_for(conn)) - table.create(bind=conn, checkfirst=True) - conn.execute( - table.delete().where(table.c.database_name == database_name, table.c.role == role) - ) - - -def test_reason_is_mandatory(cli_stack): - result = runner.invoke( - app, - ["acknowledge-schema-migration", "--database", "cli_rectify_db", "--new-schema", "whatever"], - ) - assert result.exit_code != 0 - assert "--reason" in result.output - - -def test_acknowledge_writes_a_provenance_row(cli_stack, pg_engine): - role = "vocab" - schema = f"cli_ack_{uuid.uuid4().hex[:8]}" - try: - result = runner.invoke( - app, - [ - "acknowledge-schema-migration", - "--database", "cli_rectify_db", - "--role", role, - "--new-schema", schema, - "--reason", "regression test", - ], - ) - assert result.exit_code == 0, result.stdout - assert "Acknowledged" in result.stdout - - with pg_engine.connect() as conn: - table = _schema_provenance_table(_provenance_schema_for(conn)) - row = conn.execute( - sa.select(table.c.resolved_schema, table.c.reason).where( - table.c.database_name == "cli_rectify_db", table.c.role == role - ) - ).first() - assert row is not None - assert row.resolved_schema == schema - assert row.reason == "regression test" - finally: - _cleanup_provenance_row(pg_engine, database_name="cli_rectify_db", role=role) - - -def test_drop_orphan_refuses_against_a_databases_own_current_schema(cli_stack): - result = runner.invoke( - app, - ["drop-orphan-schema-tables", "--database", "cli_rectify_db", "--schema", "public"], - ) - assert result.exit_code != 0 - assert "current schema target" in result.stdout - - -def test_drop_orphan_previews_without_confirm(cli_stack, pg_engine): - with isolated_test_schema(pg_engine, prefix="cli_drop_preview") as schema: - with pg_engine.begin() as conn: - conn.exec_driver_sql(f'CREATE TABLE "{schema}".orphaned (id int)') - - result = runner.invoke( - app, ["drop-orphan-schema-tables", "--database", "cli_rectify_db", "--schema", schema] - ) - assert result.exit_code == 0, result.stdout - assert "Would drop" in result.stdout - assert "Preview only" in result.stdout - - with pg_engine.connect() as conn: - assert sa.inspect(conn).has_table("orphaned", schema=schema) - - -def test_drop_orphan_confirm_actually_drops(cli_stack, pg_engine): - with isolated_test_schema(pg_engine, prefix="cli_drop_confirm") as schema: - with pg_engine.begin() as conn: - conn.exec_driver_sql(f'CREATE TABLE "{schema}".orphaned (id int)') - - result = runner.invoke( - app, - [ - "drop-orphan-schema-tables", - "--database", "cli_rectify_db", - "--schema", schema, - "--confirm", - ], - ) - assert result.exit_code == 0, result.stdout - assert "Dropped" in result.stdout - - with pg_engine.connect() as conn: - assert not sa.inspect(conn).has_table("orphaned", schema=schema) From 7e558a94f0320c200d63961506a7dabb68d108ca Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 00:31:15 +0000 Subject: [PATCH 15/32] Symmetrical schema tagging --- omop_alchemy/cdm/model/derived/cohort_definition.py | 2 +- omop_alchemy/cdm/model/derived/condition_era.py | 2 +- omop_alchemy/cdm/model/derived/dose_era.py | 2 +- omop_alchemy/cdm/model/derived/drug_era.py | 2 +- omop_alchemy/cdm/model/vocabulary/concept.py | 2 +- omop_alchemy/cdm/model/vocabulary/concept_ancestor.py | 2 +- omop_alchemy/cdm/model/vocabulary/concept_class.py | 2 +- omop_alchemy/cdm/model/vocabulary/concept_relationship.py | 2 +- omop_alchemy/cdm/model/vocabulary/concept_synonym.py | 2 +- omop_alchemy/cdm/model/vocabulary/drug_strength.py | 2 +- omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/omop_alchemy/cdm/model/derived/cohort_definition.py b/omop_alchemy/cdm/model/derived/cohort_definition.py index 08cf272..abddab9 100644 --- a/omop_alchemy/cdm/model/derived/cohort_definition.py +++ b/omop_alchemy/cdm/model/derived/cohort_definition.py @@ -16,9 +16,9 @@ class Cohort_Definition(CDMTableBase, Base): __tablename__ = "cohort_definition" __table_args__ = merge_table_args( + {"schema": Role.RESULTS.value}, omop_index(__tablename__, "definition_type_concept_id"), omop_index(__tablename__, "subject_concept_id"), - {"schema": Role.RESULTS.value}, ) cohort_definition_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/derived/condition_era.py b/omop_alchemy/cdm/model/derived/condition_era.py index e1072d5..0d2b7ab 100644 --- a/omop_alchemy/cdm/model/derived/condition_era.py +++ b/omop_alchemy/cdm/model/derived/condition_era.py @@ -17,9 +17,9 @@ class Condition_Era(CDMTableBase, Base): __tablename__ = "condition_era" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "condition_concept_id"), - {"schema": Role.PRIMARY.value}, ) condition_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/derived/dose_era.py b/omop_alchemy/cdm/model/derived/dose_era.py index 2b1246a..44cd213 100644 --- a/omop_alchemy/cdm/model/derived/dose_era.py +++ b/omop_alchemy/cdm/model/derived/dose_era.py @@ -16,10 +16,10 @@ class Dose_Era(CDMTableBase, Base): __tablename__ = "dose_era" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), omop_index(__tablename__, "unit_concept_id"), - {"schema": Role.PRIMARY.value}, ) dose_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/derived/drug_era.py b/omop_alchemy/cdm/model/derived/drug_era.py index c0beb61..26a483e 100644 --- a/omop_alchemy/cdm/model/derived/drug_era.py +++ b/omop_alchemy/cdm/model/derived/drug_era.py @@ -17,9 +17,9 @@ class Drug_Era(CDMTableBase, Base): __tablename__ = "drug_era" __table_args__ = merge_table_args( + {"schema": Role.PRIMARY.value}, omop_index(__tablename__, "person_id", cluster=True), omop_index(__tablename__, "drug_concept_id"), - {"schema": Role.PRIMARY.value}, ) drug_era_id: so.Mapped[int] = so.mapped_column(primary_key=True) diff --git a/omop_alchemy/cdm/model/vocabulary/concept.py b/omop_alchemy/cdm/model/vocabulary/concept.py index 202c3f0..6056f0b 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept.py +++ b/omop_alchemy/cdm/model/vocabulary/concept.py @@ -39,6 +39,7 @@ class Concept( ): __tablename__ = "concept" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "concept_code"), omop_index(__tablename__, "vocabulary_id"), omop_index(__tablename__, "domain_id"), @@ -51,7 +52,6 @@ class Concept( name="ix_concept_concept_name_lower", ), omop_table_options(cluster_on=omop_primary_key_index_name("concept")), - {"schema": Role.VOCAB.value}, ) concept_id: so.Mapped[int] = so.mapped_column(primary_key=True) concept_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py b/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py index 2d7be7c..d3d8c37 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_ancestor.py @@ -15,9 +15,9 @@ class Concept_Ancestor(Base, ReferenceTable, CDMTableBase): __tablename__ = "concept_ancestor" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "ancestor_concept_id", cluster=True), omop_index(__tablename__, "descendant_concept_id"), - {"schema": Role.VOCAB.value}, ) ancestor_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) descendant_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) diff --git a/omop_alchemy/cdm/model/vocabulary/concept_class.py b/omop_alchemy/cdm/model/vocabulary/concept_class.py index 05b6f74..816811c 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_class.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_class.py @@ -16,8 +16,8 @@ class Concept_Class(Base, ReferenceTable, CDMTableBase): __tablename__ = "concept_class" __table_args__ = merge_table_args( - omop_table_options(cluster_on=omop_primary_key_index_name("concept_class")), {"schema": Role.VOCAB.value}, + omop_table_options(cluster_on=omop_primary_key_index_name("concept_class")), ) concept_class_id: so.Mapped[str] = so.mapped_column(sa.String(20), primary_key=True) concept_class_name: so.Mapped[str] = so.mapped_column(sa.String(255), nullable=False) diff --git a/omop_alchemy/cdm/model/vocabulary/concept_relationship.py b/omop_alchemy/cdm/model/vocabulary/concept_relationship.py index ca76eb0..e6fe352 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_relationship.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_relationship.py @@ -22,10 +22,10 @@ class Concept_Relationship( ): __tablename__ = "concept_relationship" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "concept_id_1", cluster=True), omop_index(__tablename__, "concept_id_2"), omop_index(__tablename__, "relationship_id"), - {"schema": Role.VOCAB.value}, ) concept_id_1: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) concept_id_2: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) diff --git a/omop_alchemy/cdm/model/vocabulary/concept_synonym.py b/omop_alchemy/cdm/model/vocabulary/concept_synonym.py index 762c5ac..1553b7b 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept_synonym.py +++ b/omop_alchemy/cdm/model/vocabulary/concept_synonym.py @@ -15,6 +15,7 @@ class Concept_Synonym(Base, ReferenceTable, CDMTableBase): __tablename__ = "concept_synonym" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "concept_id", cluster=True), # Has to be wrapped in func.lower() as that is the common query # as it prevents captialisation mismatches between query and data. @@ -23,7 +24,6 @@ class Concept_Synonym(Base, ReferenceTable, CDMTableBase): sa.func.lower(sa.column("concept_synonym_name")), name="ix_concept_synonym_concept_synonym_name_lower", ), - {"schema": Role.VOCAB.value}, ) concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) concept_synonym_name: so.Mapped[str] = so.mapped_column(sa.String(1000),primary_key=True) diff --git a/omop_alchemy/cdm/model/vocabulary/drug_strength.py b/omop_alchemy/cdm/model/vocabulary/drug_strength.py index f3ccdd3..10feb2f 100644 --- a/omop_alchemy/cdm/model/vocabulary/drug_strength.py +++ b/omop_alchemy/cdm/model/vocabulary/drug_strength.py @@ -30,9 +30,9 @@ class Drug_Strength( """ __tablename__ = "drug_strength" __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "drug_concept_id", cluster=True), omop_index(__tablename__, "ingredient_concept_id"), - {"schema": Role.VOCAB.value}, ) drug_concept_id: so.Mapped[int] = so.mapped_column(sa.ForeignKey(role_fk(Role.VOCAB, "concept.concept_id")),primary_key=True) diff --git a/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py b/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py index 9516348..bebc0b9 100644 --- a/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py +++ b/omop_alchemy/cdm/model/vocabulary/source_to_concept_map.py @@ -34,11 +34,11 @@ class Source_To_Concept_Map( __tablename__ = "source_to_concept_map" __cdm_extra_checks__ = ["source_concept_id_range"] __table_args__ = merge_table_args( + {"schema": Role.VOCAB.value}, omop_index(__tablename__, "target_concept_id", cluster=True), omop_index(__tablename__, "source_vocabulary_id"), omop_index(__tablename__, "target_vocabulary_id"), omop_index(__tablename__, "source_code"), - {"schema": Role.VOCAB.value}, ) source_code: so.Mapped[str] = so.mapped_column(sa.String(50),primary_key=True) From ed709a77b975ca254d3c42bfe9275ae91dd95708 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 00:49:32 +0000 Subject: [PATCH 16/32] Update schema wiring --- omop_alchemy/backends/base.py | 31 ++++++- omop_alchemy/backends/postgres.py | 65 +++++++++----- omop_alchemy/backends/sqlite.py | 5 +- omop_alchemy/config.py | 48 ++++------ omop_alchemy/maintenance/cli_foreign_keys.py | 50 ++++++----- omop_alchemy/maintenance/cli_fulltext.py | 10 +-- omop_alchemy/maintenance/cli_indexes.py | 45 ++++++---- omop_alchemy/maintenance/cli_schema.py | 83 +---------------- omop_alchemy/maintenance/cli_schema_doctor.py | 6 +- omop_alchemy/maintenance/cli_schema_info.py | 1 - .../maintenance/cli_schema_reconcile.py | 1 + .../maintenance/cli_schema_summary.py | 9 +- omop_alchemy/maintenance/cli_schema_tables.py | 32 ++++--- omop_alchemy/maintenance/cli_tables.py | 90 ++++++++++++------- omop_alchemy/maintenance/cli_vocab.py | 5 -- omop_alchemy/maintenance/tables.py | 43 +++++++-- ...st_backends_non_default_schema_postgres.py | 24 ++--- tests/test_foreign_keys.py | 24 +++-- tests/test_fulltext.py | 14 +-- tests/test_indexes.py | 18 ++-- tests/test_inspect_functional_index_probe.py | 25 ++++++ tests/test_load_vocab_source.py | 4 +- tests/test_schema_doctor.py | 3 - tests/test_schema_provenance_guard.py | 2 - tests/test_schema_reconcile.py | 31 ++++--- tests/test_truncate_tables.py | 5 +- ...test_vocab_results_role_wiring_postgres.py | 64 ++++++++++++- 27 files changed, 435 insertions(+), 303 deletions(-) create mode 100644 tests/test_inspect_functional_index_probe.py diff --git a/omop_alchemy/backends/base.py b/omop_alchemy/backends/base.py index f5a72ab..38e68cb 100644 --- a/omop_alchemy/backends/base.py +++ b/omop_alchemy/backends/base.py @@ -3,6 +3,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING, Any import sqlalchemy as sa +from oa_configurator import Role if TYPE_CHECKING: from sqlalchemy.sql import ColumnElement @@ -85,6 +86,7 @@ def toggle_fk_triggers( table_name: str, *, enable: bool, + role: Role = Role.PRIMARY, ) -> None: raise FeatureNotSupportedError("FK trigger management", self) @@ -92,6 +94,8 @@ def get_fk_trigger_counts( self, conn: sa.Connection, table_name: str, + *, + role: Role = Role.PRIMARY, ) -> tuple[int, int]: """Return (disabled_count, enabled_count) for RI triggers on the table.""" raise FeatureNotSupportedError("FK trigger status inspection", self) @@ -103,6 +107,9 @@ def count_fk_violations( referred_table: str, constrained_cols: list[str], referred_cols: list[str], + *, + source_role: Role = Role.PRIMARY, + referred_role: Role = Role.PRIMARY, ) -> int: raise FeatureNotSupportedError("FK constraint violation counting", self) @@ -113,6 +120,8 @@ def cluster_table( conn: sa.Connection, table_name: str, index_name: str, + *, + role: Role = Role.PRIMARY, ) -> None: raise FeatureNotSupportedError("Table clustering", self) @@ -120,6 +129,8 @@ def get_clustered_index_name( self, conn: sa.Connection, table_name: str, + *, + role: Role = Role.PRIMARY, ) -> str | None: raise FeatureNotSupportedError("Cluster index inspection", self) @@ -147,12 +158,15 @@ def analyze_table( table_name: str, *, vacuum: bool = False, + role: Role = Role.PRIMARY, ) -> None: ... def index_exists( self, conn: sa.Connection, index_name: str, + *, + role: Role = Role.PRIMARY, ) -> bool: """Return True when the named index currently exists on the database. @@ -165,12 +179,17 @@ def drop_index_if_exists( self, conn: sa.Connection, index_name: str, + *, + role: Role = Role.PRIMARY, ) -> None: """Drop an index by name without relying on SQLAlchemy's reflection-based checkfirst. Some backends (e.g. SQLite) can't reflect expression-based indexes, so Index.drop(checkfirst=True) would silently no-op on them. IF EXISTS is - evaluated by the database itself, not by reflection. + evaluated by the database itself, not by reflection. role is accepted + for interface parity with the schema-aware override + (PostgresBackend); this default implementation is unqualified, + matching SQLite having no schema concept to route through. """ conn.exec_driver_sql(f'DROP INDEX IF EXISTS "{index_name}"') @@ -181,6 +200,7 @@ def truncate_table_batch( *, restart_identities: bool, cascade: bool, + role: Role = Role.PRIMARY, ) -> None: raise FeatureNotSupportedError("TRUNCATE with RESTART IDENTITY / CASCADE", self) @@ -191,6 +211,8 @@ def find_sequence_name( conn: sa.Connection, table_name: str, column_name: str, + *, + role: Role = Role.PRIMARY, ) -> str | None: raise FeatureNotSupportedError("Owned sequence lookup", self) @@ -238,6 +260,7 @@ def install_fulltext_on_table( index_name: str, create_indexes: bool, fastupdate: bool, + role: Role = Role.PRIMARY, ) -> None: raise FeatureNotSupportedError("Full-text search", self) @@ -249,6 +272,7 @@ def populate_fulltext_on_table( vector_column_name: str, source_column_name: str, regconfig: str, + role: Role = Role.PRIMARY, ) -> int | None: raise FeatureNotSupportedError("Full-text search", self) @@ -260,6 +284,7 @@ def drop_fulltext_on_table( vector_column_name: str, index_name: str, drop_indexes: bool, + role: Role = Role.PRIMARY, ) -> None: raise FeatureNotSupportedError("Full-text search", self) @@ -270,6 +295,8 @@ def prepare_backup( engine: sa.Engine, output_path: str, backup_format: str, + *, + role: Role = Role.PRIMARY, ) -> tuple[str, list[str], dict[str, str], str]: """Return (tool_path, command, env, database_name). subprocess.run stays in CLI.""" raise FeatureNotSupportedError("Database backup", self) @@ -279,6 +306,8 @@ def prepare_restore( engine: sa.Engine, input_path: str, backup_format: str, + *, + role: Role = Role.PRIMARY, ) -> tuple[str, list[str], dict[str, str], str]: """Return (tool_path, command, env, database_name). subprocess.run stays in CLI.""" raise FeatureNotSupportedError("Database restore", self) diff --git a/omop_alchemy/backends/postgres.py b/omop_alchemy/backends/postgres.py index 6a964f9..4b6b17d 100644 --- a/omop_alchemy/backends/postgres.py +++ b/omop_alchemy/backends/postgres.py @@ -5,7 +5,7 @@ import sqlalchemy as sa -from oa_configurator import Dialect, qualified, schema_of +from oa_configurator import Dialect, Role, qualified, schema_of from sqlalchemy.dialects.postgresql import REGCONFIG, TSVECTOR from sqlalchemy.sql import func @@ -30,16 +30,19 @@ def toggle_fk_triggers( table_name: str, *, enable: bool, + role: Role = Role.PRIMARY, ) -> None: action = "ENABLE" if enable else "DISABLE" conn.exec_driver_sql( - f"ALTER TABLE {qualified(conn, table_name)} {action} TRIGGER ALL" + f"ALTER TABLE {qualified(conn, table_name, role=role)} {action} TRIGGER ALL" ) def get_fk_trigger_counts( self, conn: sa.Connection, table_name: str, + *, + role: Role = Role.PRIMARY, ) -> tuple[int, int]: disabled_count, enabled_count = conn.execute( sa.text( @@ -56,7 +59,7 @@ def get_fk_trigger_counts( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": schema_of(conn)}, + {"table_name": table_name, "db_schema": schema_of(conn, role=role)}, ).one() return int(disabled_count or 0), int(enabled_count or 0) @@ -67,9 +70,12 @@ def count_fk_violations( referred_table: str, constrained_cols: list[str], referred_cols: list[str], + *, + source_role: Role = Role.PRIMARY, + referred_role: Role = Role.PRIMARY, ) -> int: - source = qualified(conn, source_table) - referred = qualified(conn, referred_table) + source = qualified(conn, source_table, role=source_role) + referred = qualified(conn, referred_table, role=referred_role) non_null_predicate = " AND ".join( f"src.{col} IS NOT NULL" for col in constrained_cols ) @@ -99,15 +105,19 @@ def cluster_table( conn: sa.Connection, table_name: str, index_name: str, + *, + role: Role = Role.PRIMARY, ) -> None: conn.exec_driver_sql( - f"CLUSTER {qualified(conn, table_name)} USING {index_name}" + f"CLUSTER {qualified(conn, table_name, role=role)} USING {index_name}" ) def get_clustered_index_name( self, conn: sa.Connection, table_name: str, + *, + role: Role = Role.PRIMARY, ) -> str | None: result = conn.execute( sa.text( @@ -122,7 +132,7 @@ def get_clustered_index_name( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": schema_of(conn)}, + {"table_name": table_name, "db_schema": schema_of(conn, role=role)}, ).scalar_one_or_none() return str(result) if result is not None else None @@ -149,16 +159,19 @@ def analyze_table( table_name: str, *, vacuum: bool = False, + role: Role = Role.PRIMARY, ) -> None: operation = "VACUUM ANALYZE" if vacuum else "ANALYZE" - conn.exec_driver_sql(f"{operation} {qualified(conn, table_name)}") + conn.exec_driver_sql(f"{operation} {qualified(conn, table_name, role=role)}") def index_exists( self, conn: sa.Connection, index_name: str, + *, + role: Role = Role.PRIMARY, ) -> bool: - qualified_index_name = qualified(conn, index_name) + qualified_index_name = qualified(conn, index_name, role=role) return bool( conn.scalar( sa.select( @@ -167,8 +180,10 @@ def index_exists( ) ) - def drop_index_if_exists(self, conn: sa.Connection, index_name: str) -> None: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name)}") + def drop_index_if_exists( + self, conn: sa.Connection, index_name: str, *, role: Role = Role.PRIMARY + ) -> None: + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, role=role)}") def truncate_table_batch( self, @@ -177,9 +192,10 @@ def truncate_table_batch( *, restart_identities: bool, cascade: bool, + role: Role = Role.PRIMARY, ) -> None: sql = "TRUNCATE TABLE " + ", ".join( - qualified(conn, name) for name in table_names + qualified(conn, name, role=role) for name in table_names ) if restart_identities: sql += " RESTART IDENTITY" @@ -194,8 +210,10 @@ def find_sequence_name( conn: sa.Connection, table_name: str, column_name: str, + *, + role: Role = Role.PRIMARY, ) -> str | None: - fully_qualified = qualified(conn, table_name) + fully_qualified = qualified(conn, table_name, role=role) return conn.execute( sa.text("SELECT pg_get_serial_sequence(:table_name, :column_name)"), {"table_name": fully_qualified, "column_name": column_name}, @@ -283,8 +301,9 @@ def install_fulltext_on_table( index_name: str, create_indexes: bool, fastupdate: bool, + role: Role = Role.PRIMARY, ) -> None: - qualified_table = qualified(conn, table_name) + qualified_table = qualified(conn, table_name, role=role) conn.exec_driver_sql( f"ALTER TABLE {qualified_table} ADD COLUMN IF NOT EXISTS {vector_column_name} tsvector" ) @@ -293,7 +312,7 @@ def install_fulltext_on_table( table_name, sa.MetaData(), sa.Column(vector_column_name, TSVECTOR), - schema=schema_of(conn), + schema=schema_of(conn, role=role), ) index = sa.Index( index_name, @@ -311,12 +330,13 @@ def populate_fulltext_on_table( vector_column_name: str, source_column_name: str, regconfig: str, + role: Role = Role.PRIMARY, ) -> int | None: lightweight_table = sa.table( table_name, sa.column(vector_column_name), sa.column(source_column_name), - schema=schema_of(conn), + schema=schema_of(conn, role=role), ) source_column = lightweight_table.c[source_column_name] stmt = lightweight_table.update().values( @@ -340,11 +360,12 @@ def drop_fulltext_on_table( vector_column_name: str, index_name: str, drop_indexes: bool, + role: Role = Role.PRIMARY, ) -> None: if drop_indexes: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name)}") + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, role=role)}") conn.exec_driver_sql( - f"ALTER TABLE {qualified(conn, table_name)}" + f"ALTER TABLE {qualified(conn, table_name, role=role)}" f" DROP COLUMN IF EXISTS {vector_column_name}" ) @@ -355,6 +376,8 @@ def prepare_backup( engine: sa.Engine, output_path: str, backup_format: str, + *, + role: Role = Role.PRIMARY, ) -> tuple[str, list[str], dict[str, str], str]: tool_path = _pg_dump_path() url = engine.url @@ -373,7 +396,7 @@ def prepare_backup( "--no-owner", "--no-privileges", ] - db_schema = schema_of(engine) + db_schema = schema_of(engine, role=role) if db_schema: command.extend(["--schema", db_schema]) env = os.environ.copy() @@ -386,6 +409,8 @@ def prepare_restore( engine: sa.Engine, input_path: str, backup_format: str, + *, + role: Role = Role.PRIMARY, ) -> tuple[str, list[str], dict[str, str], str]: url = engine.url database_name = url.database @@ -394,7 +419,7 @@ def prepare_restore( "Database restore requires a database name in the configured engine URL." ) connection_uri = _libpq_connection_uri(url) - db_schema = schema_of(engine) + db_schema = schema_of(engine, role=role) if backup_format == "custom": tool_path = _pg_restore_path() diff --git a/omop_alchemy/backends/sqlite.py b/omop_alchemy/backends/sqlite.py index 1803c8b..f318062 100644 --- a/omop_alchemy/backends/sqlite.py +++ b/omop_alchemy/backends/sqlite.py @@ -1,7 +1,7 @@ from __future__ import annotations import sqlalchemy as sa -from oa_configurator import Dialect +from oa_configurator import Dialect, Role from .base import Backend, FeatureNotSupportedError @@ -20,6 +20,8 @@ def index_exists( self, conn: sa.Connection, index_name: str, + *, + role: Role = Role.PRIMARY, ) -> bool: row = conn.exec_driver_sql( "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?", @@ -33,6 +35,7 @@ def analyze_table( table_name: str, *, vacuum: bool = False, + role: Role = Role.PRIMARY, ) -> None: if vacuum: raise FeatureNotSupportedError("VACUUM ANALYZE", self) diff --git a/omop_alchemy/config.py b/omop_alchemy/config.py index b08ce60..a36e5b2 100644 --- a/omop_alchemy/config.py +++ b/omop_alchemy/config.py @@ -110,39 +110,21 @@ def get_cdm_context() -> tuple[OmopAlchemyConfig, ResolvedCDMDatabase]: def vocabulary_identity(resolved: ResolvedCDMDatabase) -> str | None: """Stable identity for the vocabulary dataset ``resolved`` reads, or None. - Concept-set expansions are a function of the vocabulary, so caching them - against this identity means recreating an engine against the same dataset - reuses the expansion instead of re-running ``concept_ancestor`` traversals. - - Composed from the **vocab** role rather than the primary one, because - ``concept_ancestor`` is a vocabulary table. On any deployment that does not - configure a separate vocabulary target this resolves to the CDM database, so - it costs nothing today and stays correct if vocabulary routing is ever - honoured by the ORM. Do not "simplify" it to ``resolved.connection``. - - Uses ``safe_url``, the credential-redacted form, so no password reaches a - cache key. - - **Returns None wherever sharing would be unsafe, so every caller inherits - that judgement.** Exported precisely so that packages building their own - engines compose the identity the same way — two spellings of one dataset - would produce two cache entries that each look authoritative. That only works - if the safety conditions live here rather than at one call site. - - Two conditions yield None: - - *Split vocabulary target.* Vocabulary models use the primary logical schema, - and one SQLAlchemy engine cannot route tables to a second physical - connection, so a declared vocabulary target that differs from the primary is - not what the engine actually reads. Returning its identity would let two - different primary databases that name the same external vocabulary share - expansions — one database's concept sets served for another. Such a - deployment falls back to per-engine caching until ORM routing supports it. - - *Ephemeral database.* In-memory SQLite, where two engines built from - identical configuration are genuinely separate databases. - - Both cases are correct-but-unshared rather than wrong. + Caches concept-set expansions (``concept_ancestor`` traversals) across + engines reading the same vocabulary. Built from the VOCAB role, not + primary, since ``concept_ancestor`` is a vocabulary table; do not + simplify to ``resolved.connection``. Uses ``safe_url`` so no password + reaches the cache key. + + Returns None wherever sharing would be unsafe, so every caller inherits + that judgement instead of each composing its own identity: + - a split vocabulary target: schema_translate_map cannot route to a different + physical connection, so identity based on the declared target would not + match what the engine actually reads, or + - an ephemeral database: in-memory SQLite, where identically-configured engines are genuinely + separate databases. + + Both fall back to per-engine caching instead of being wrong. """ vocab_target = resolved.connection_target(Role.VOCAB) diff --git a/omop_alchemy/maintenance/cli_foreign_keys.py b/omop_alchemy/maintenance/cli_foreign_keys.py index 2e3ce91..88ec91a 100644 --- a/omop_alchemy/maintenance/cli_foreign_keys.py +++ b/omop_alchemy/maintenance/cli_foreign_keys.py @@ -7,6 +7,7 @@ import sqlalchemy as sa import typer +from oa_configurator import Role, schema_of from ..backends import Backend, resolve_backend, require_backend_support, backend_support_note from ._cli_utils import Status, dry_label, dry_status, omop_command from .tables import ( @@ -32,6 +33,7 @@ class ForeignKeyBase: table_name: str category: TableCategory + role: Role @dataclass(frozen=True) @@ -87,24 +89,25 @@ class ForeignKeyValidationReport: def _collect_fk_info( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, ) -> list[_FKTableInfo]: """Return all ORM-managed tables that participate in at least one FK relationship (outgoing or incoming).""" inspector = sa.inspect(engine) selected_tables = existing_maintenance_tables( - inspector, - db_schema=db_schema, + engine, vocabulary_included=vocabulary_included, ) - selected_names = {table.table_name for table in selected_tables} + tables_by_name = {table.table_name: table for table in selected_tables} + selected_names = set(tables_by_name) incoming_counts = {name: 0 for name in selected_names} outgoing_counts = {name: 0 for name in selected_names} for table_name in selected_names: - foreign_keys = inspector.get_foreign_keys(table_name, schema=db_schema) + foreign_keys = inspector.get_foreign_keys( + table_name, schema=schema_of(engine, role=tables_by_name[table_name].role) + ) relevant_foreign_keys = [ foreign_key for foreign_key in foreign_keys @@ -128,6 +131,7 @@ def _collect_fk_info( _FKTableInfo( table_name=table.table_name, category=table.category, + role=table.role, outgoing_constraint_count=outgoing_count, incoming_constraint_count=incoming_count, ) @@ -140,7 +144,6 @@ def _collect_strict_validation_failures( connection: sa.Connection, backend: Backend, *, - db_schema: str | None, vocabulary_included: bool, ) -> dict[str, list[ForeignKeyConstraintViolation]]: """Query every FK constraint across selected tables and return a mapping of table name → violation list. @@ -150,18 +153,21 @@ def _collect_strict_validation_failures( """ inspector = sa.inspect(connection) selected_tables = existing_maintenance_tables( - inspector, - db_schema=db_schema, + connection, vocabulary_included=vocabulary_included, ) - selected_names = {table.table_name for table in selected_tables} + tables_by_name = {table.table_name: table for table in selected_tables} + selected_names = set(tables_by_name) failures: dict[str, list[ForeignKeyConstraintViolation]] = { table_name: [] for table_name in selected_names } for table_name in sorted(selected_names): - for foreign_key in inspector.get_foreign_keys(table_name, schema=db_schema): + source_role = tables_by_name[table_name].role + for foreign_key in inspector.get_foreign_keys( + table_name, schema=schema_of(connection, role=source_role) + ): referred_table = foreign_key.get("referred_table") constrained_columns = foreign_key.get("constrained_columns") or [] referred_columns = foreign_key.get("referred_columns") or [] @@ -179,6 +185,8 @@ def _collect_strict_validation_failures( str(referred_table), list(constrained_columns), list(referred_columns), + source_role=source_role, + referred_role=tables_by_name[str(referred_table)].role, ) if violation_count == 0: @@ -220,7 +228,6 @@ def _fk_violation_detail( def validate_foreign_key_constraints( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, ) -> ForeignKeyValidationReport: """Count rows that violate each FK constraint and return a full per-table validation report.""" @@ -229,7 +236,6 @@ def validate_foreign_key_constraints( targets = _collect_fk_info( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) @@ -237,7 +243,6 @@ def validate_foreign_key_constraints( validation_failures = _collect_strict_validation_failures( connection, backend, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) @@ -252,6 +257,7 @@ def validate_foreign_key_constraints( ForeignKeyValidationResult( table_name=target.table_name, category=target.category, + role=target.role, outgoing_constraint_count=target.outgoing_constraint_count, incoming_constraint_count=target.incoming_constraint_count, violating_constraint_count=violating_constraint_count, @@ -279,7 +285,6 @@ def manage_foreign_key_triggers( engine: sa.Engine, *, enable: bool = False, - db_schema: str | None = None, vocabulary_included: bool = False, dry_run: bool = False, strict: bool = False, @@ -290,7 +295,6 @@ def manage_foreign_key_triggers( targets = _collect_fk_info( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) @@ -300,7 +304,6 @@ def manage_foreign_key_triggers( validation_failures = _collect_strict_validation_failures( connection, backend, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) if validation_failures: @@ -310,6 +313,7 @@ def manage_foreign_key_triggers( ForeignKeyManagementResult( table_name=target.table_name, category=target.category, + role=target.role, outgoing_constraint_count=target.outgoing_constraint_count, incoming_constraint_count=target.incoming_constraint_count, enable=enable, @@ -331,12 +335,15 @@ def manage_foreign_key_triggers( } for target in targets: if not dry_run: - backend.toggle_fk_triggers(connection, target.table_name, enable=enable) + backend.toggle_fk_triggers( + connection, target.table_name, enable=enable, role=target.role + ) results.append( ForeignKeyManagementResult( table_name=target.table_name, category=target.category, + role=target.role, outgoing_constraint_count=target.outgoing_constraint_count, incoming_constraint_count=target.incoming_constraint_count, enable=enable, @@ -351,7 +358,6 @@ def manage_foreign_key_triggers( def collect_foreign_key_trigger_status( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, ) -> list[ForeignKeyStatusResult]: """Query pg_trigger to count disabled vs enabled RI triggers for each participating table.""" @@ -360,7 +366,6 @@ def collect_foreign_key_trigger_status( targets = _collect_fk_info( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) results: list[ForeignKeyStatusResult] = [] @@ -368,12 +373,13 @@ def collect_foreign_key_trigger_status( with engine.connect() as connection: for target in targets: disabled_count, enabled_count = backend.get_fk_trigger_counts( - connection, target.table_name + connection, target.table_name, role=target.role ) results.append( ForeignKeyStatusResult( table_name=target.table_name, category=target.category, + role=target.role, disabled_trigger_count=disabled_count, enabled_trigger_count=enabled_count, outgoing_constraint_count=target.outgoing_constraint_count, @@ -415,7 +421,6 @@ def disable_foreign_keys_command( results = manage_foreign_key_triggers( engine, enable=False, - db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, strict=strict, @@ -452,7 +457,6 @@ def enable_foreign_keys_command( results = manage_foreign_key_triggers( engine, enable=True, - db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, strict=strict, @@ -477,7 +481,6 @@ def foreign_key_status_command( with console.status("Inspecting foreign key trigger status..."): results = collect_foreign_key_trigger_status( engine, - db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, ) console.print(render_foreign_key_status_results(results)) @@ -499,7 +502,6 @@ def foreign_key_validate_command( with console.status("Validating selected foreign key relationships..."): report = validate_foreign_key_constraints( engine, - db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, ) console.print(render_foreign_key_validation_results(report.results)) diff --git a/omop_alchemy/maintenance/cli_fulltext.py b/omop_alchemy/maintenance/cli_fulltext.py index b3c9046..6c41113 100644 --- a/omop_alchemy/maintenance/cli_fulltext.py +++ b/omop_alchemy/maintenance/cli_fulltext.py @@ -8,6 +8,7 @@ import typer from sqlalchemy.engine import Engine +from oa_configurator import Role from ..backends import backend_support_note as _backend_support_note from ..backends import resolve_backend, require_backend_support from ..backends.base import FullTextError @@ -51,7 +52,6 @@ class FullTextResult: def install_fulltext_columns( engine: Engine, *, - db_schema: str | None = None, create_indexes: bool = True, fastupdate: bool = False, dry_run: bool = False, @@ -72,6 +72,7 @@ def install_fulltext_columns( index_name=cfg.index_name, create_indexes=create_indexes, fastupdate=fastupdate, + role=Role.VOCAB, ) backend.register_fulltext_metadata() except FullTextError: @@ -103,7 +104,6 @@ def install_fulltext_columns( def populate_fulltext_columns( engine: Engine, *, - db_schema: str | None = None, regconfig: str = "english", dry_run: bool = False, ) -> tuple[FullTextResult, ...]: @@ -123,6 +123,7 @@ def populate_fulltext_columns( vector_column_name=cfg.vector_column_name, source_column_name=cfg.source_column_name, regconfig=regconfig, + role=Role.VOCAB, ) backend.register_fulltext_metadata() except FullTextError: @@ -151,7 +152,6 @@ def populate_fulltext_columns( def drop_fulltext_columns( engine: Engine, *, - db_schema: str | None = None, drop_indexes: bool = True, dry_run: bool = False, ) -> tuple[FullTextResult, ...]: @@ -170,6 +170,7 @@ def drop_fulltext_columns( vector_column_name=cfg.vector_column_name, index_name=cfg.index_name, drop_indexes=drop_indexes, + role=Role.VOCAB, ) backend.unregister_fulltext_metadata() except FullTextError: @@ -221,7 +222,6 @@ def install_fulltext_command( with console.status("Managing PostgreSQL full-text sidecar columns..."): results = install_fulltext_columns( engine, - db_schema=conn.resolved.schema_name, create_indexes=create_indexes, fastupdate=fastupdate, dry_run=dry_run, @@ -245,7 +245,6 @@ def populate_fulltext_command( with console.status("Managing PostgreSQL full-text sidecar columns..."): results = populate_fulltext_columns( engine, - db_schema=conn.resolved.schema_name, regconfig=regconfig, dry_run=dry_run, ) @@ -269,7 +268,6 @@ def drop_fulltext_command( with console.status("Managing PostgreSQL full-text sidecar columns..."): results = drop_fulltext_columns( engine, - db_schema=conn.resolved.schema_name, drop_indexes=drop_indexes, dry_run=dry_run, ) diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index 8b57343..23fc32a 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -10,7 +10,7 @@ from sqlalchemy.exc import DBAPIError, IntegrityError import typer -from oa_configurator import ensure_schema, supports_schemas +from oa_configurator import Role, ensure_schema, schema_of, supports_schemas from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY @@ -36,6 +36,7 @@ class IndexTarget: table_name: str category: TableCategory + role: Role index_name: str column_names: tuple[str, ...] unique: bool @@ -585,7 +586,6 @@ def _resolve_physical_cluster_name( def collect_index_targets( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, ) -> list[IndexTarget]: """List ORM-defined indexes that currently exist in the target database.""" @@ -594,10 +594,11 @@ def collect_index_targets( targets: list[IndexTarget] = [] for table in selected_tables: - if not inspector.has_table(table.table_name, schema=db_schema): + table_schema = schema_of(engine, role=table.role) + if not inspector.has_table(table.table_name, schema=table_schema): continue - existing_indexes = inspector.get_indexes(table.table_name, schema=db_schema) + existing_indexes = inspector.get_indexes(table.table_name, schema=table_schema) existing_index_names = {index["name"] for index in existing_indexes} for metadata_index in sorted(table.table.indexes, key=lambda idx: idx.name or ""): @@ -615,6 +616,7 @@ def collect_index_targets( IndexTarget( table_name=table.table_name, category=table.category, + role=table.role, index_name=physical_name, column_names=column_names, unique=unique, @@ -653,7 +655,6 @@ def manage_indexes( engine: sa.Engine, *, enable: bool, - db_schema: str | None = None, vocabulary_included: bool = False, dry_run: bool = False, cluster: bool = True, @@ -668,6 +669,7 @@ def manage_indexes( results: list[IndexManagementResult] = [] for table in selected_tables: + db_schema = schema_of(engine, role=table.role) if not inspector.has_table(table.table_name, schema=db_schema): continue @@ -715,7 +717,9 @@ def manage_indexes( with connection_factory() as connection: if not enable: if not dry_run: - existed_before_drop = backend.index_exists(connection, index_name) + existed_before_drop = backend.index_exists( + connection, index_name, role=table.role + ) else: existed_before_drop = exists if not existed_before_drop: @@ -738,7 +742,9 @@ def manage_indexes( captured = pending_capture is None if captured: if not dry_run: - backend.drop_index_if_exists(connection, equivalent_name) + backend.drop_index_if_exists( + connection, equivalent_name, role=table.role + ) outcome = _IndexOutcome( status=dry_status(dry_run, Status.CAPTURED), detail=dry_label( @@ -790,7 +796,7 @@ def manage_indexes( physical_name=index_name, ) elif not dry_run: - backend.drop_index_if_exists(connection, index_name) + backend.drop_index_if_exists(connection, index_name, role=table.role) # outcome stays default: applied / "metadata-defined index dropped" # dry-run, existed_before_drop True: outcome stays default ("would be dropped") else: @@ -856,6 +862,7 @@ def manage_indexes( operation="index", table_name=table.table_name, category=table.category, + role=table.role, index_name=physical_name, column_names=column_names, unique=unique, @@ -893,6 +900,7 @@ def manage_indexes( operation="cluster", table_name=table.table_name, category=table.category, + role=table.role, index_name=physical_cluster_name, column_names=cluster_columns, unique=False, @@ -909,7 +917,9 @@ def manage_indexes( else: if not dry_run: with engine.begin() as connection: - backend.cluster_table(connection, table.table_name, physical_cluster_name) + backend.cluster_table( + connection, table.table_name, physical_cluster_name, role=table.role + ) clustered_now = True results.append( @@ -917,6 +927,7 @@ def manage_indexes( operation="cluster", table_name=table.table_name, category=table.category, + role=table.role, index_name=physical_cluster_name, column_names=cluster_columns, unique=False, @@ -929,7 +940,7 @@ def manage_indexes( if not dry_run and (created_any or clustered_now): with engine.connect() as connection: - backend.analyze_table(connection, table.table_name) + backend.analyze_table(connection, table.table_name, role=table.role) connection.commit() return results @@ -958,7 +969,6 @@ def disable_indexes_command( results = manage_indexes( engine, enable=False, - db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, ) @@ -994,7 +1004,6 @@ def enable_indexes_command( results = manage_indexes( engine, enable=True, - db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, cluster=cluster, @@ -1035,7 +1044,8 @@ def cluster_tables_command( results: list[IndexManagementResult] = [] for table in selected_tables: - if not inspector.has_table(table.table_name, schema=conn.resolved.schema_name): + table_schema = schema_of(engine, role=table.role) + if not inspector.has_table(table.table_name, schema=table_schema): continue cluster_index_name = _cluster_target_name(table) @@ -1043,7 +1053,7 @@ def cluster_tables_command( continue cluster_columns = _cluster_column_names(table, cluster_index_name) - existing_indexes = inspector.get_indexes(table.table_name, schema=conn.resolved.schema_name) + existing_indexes = inspector.get_indexes(table.table_name, schema=table_schema) physical_cluster_name = _resolve_physical_cluster_name( existing_indexes, cluster_index_name, @@ -1052,9 +1062,11 @@ def cluster_tables_command( if not dry_run: with engine.begin() as connection: - backend.cluster_table(connection, table.table_name, physical_cluster_name) + backend.cluster_table( + connection, table.table_name, physical_cluster_name, role=table.role + ) with engine.connect() as connection: - backend.analyze_table(connection, table.table_name) + backend.analyze_table(connection, table.table_name, role=table.role) connection.commit() results.append( @@ -1062,6 +1074,7 @@ def cluster_tables_command( operation="cluster", table_name=table.table_name, category=table.category, + role=table.role, index_name=physical_cluster_name, column_names=cluster_columns, unique=False, diff --git a/omop_alchemy/maintenance/cli_schema.py b/omop_alchemy/maintenance/cli_schema.py index 01e53dc..88f4d09 100644 --- a/omop_alchemy/maintenance/cli_schema.py +++ b/omop_alchemy/maintenance/cli_schema.py @@ -3,9 +3,8 @@ from __future__ import annotations import typer -from oa_configurator import Resolver, Role, load_stack_config, record_schema_provenance -from ._cli_utils import handle_error, omop_command +from ._cli_utils import omop_command from .cli_schema_doctor import ( DoctorCheck as DoctorCheck, DoctorReport as DoctorReport, @@ -24,10 +23,6 @@ TableReconciliationResult as TableReconciliationResult, reconcile_schema, ) -from .cli_schema_rectify import ( - OrphanTablePreview as OrphanTablePreview, - drop_orphan_schema_tables, -) from .cli_schema_summary import ( TableSummaryResult as TableSummaryResult, collect_data_summary, @@ -159,7 +154,6 @@ def create_missing_tables_command( results = create_missing_tables( engine, vocab_engine=vocab_engine, - db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, resolved=conn.resolved, @@ -191,7 +185,6 @@ def data_summary_command( with console.status("Collecting table summary..."): results = collect_data_summary( engine, - db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, existing_only=not include_missing, ) @@ -199,77 +192,3 @@ def data_summary_command( console.print(render_data_summary_summary(results)) -@app.command("acknowledge-schema-migration") -def acknowledge_schema_migration_command( - database: str = typer.Option(..., "--database", help="Name of the [databases.*] entry to acknowledge."), - role: Role = typer.Option(Role.PRIMARY, "--role", help="Logical role whose schema is being acknowledged."), - new_schema: str = typer.Option(..., "--new-schema", help="Schema to record as the accepted baseline."), - reason: str = typer.Option( - ..., - "--reason", - help="Free-text justification for this acknowledgment. Mandatory: there is no --yes shortcut.", - ), -) -> None: - """Record a schema as the deliberate baseline for a database/role. - - Overwrites any existing provenance row (its prior value moves to - previous_schema); does not touch the CDM tables themselves. - """ - try: - stack = load_stack_config() - resolved = Resolver(stack).resolve_database(database) - engine = resolved.create_engine(role=role) - try: - with engine.begin() as connection: - record_schema_provenance( - connection, resolved, role=role, new_schema=new_schema, reason=reason - ) - finally: - engine.dispose() - except Exception as exc: - handle_error(exc) - console.print( - f"[green]Acknowledged[/green] {database!r} (role {role.value!r}) -> schema {new_schema!r}." - ) - - -@app.command("drop-orphan-schema-tables") -def drop_orphan_schema_tables_command( - database: str = typer.Option(..., "--database", help="Name of the [databases.*] entry providing the connection."), - schema: str = typer.Option(..., "--schema", help="Orphan schema to inspect/drop tables from."), - role: Role = typer.Option(Role.PRIMARY, "--role", help="Logical role providing the connection to use."), - confirm: bool = typer.Option( - False, - "--confirm", - help="Actually drop the previewed tables. Omit to preview only.", - ), -) -> None: - """Drop tables physically found in an orphaned schema, after a stack-wide safety check. - - Refuses if the named schema is still the current schema target of any - configured database/role, not just the one named here. Without - --confirm, only previews what would be dropped. - """ - try: - stack = load_stack_config() - resolved = Resolver(stack).resolve_database(database) - engine = resolved.create_engine(role=role) - try: - with engine.begin() as connection: - preview = drop_orphan_schema_tables( - connection, stack=stack, orphan_schema=schema, confirm=confirm - ) - finally: - engine.dispose() - - if not preview: - console.print(f"No tables found in schema {schema!r}.") - return - for item in preview: - count = "unknown" if item.approximate_row_count is None else str(item.approximate_row_count) - verb = "Dropped" if confirm else "Would drop" - console.print(f"{verb} {schema}.{item.table_name} (~{count} rows)") - if not confirm: - console.print("[yellow]Preview only. Re-run with --confirm to actually drop these tables.[/yellow]") - except Exception as exc: - handle_error(exc) diff --git a/omop_alchemy/maintenance/cli_schema_doctor.py b/omop_alchemy/maintenance/cli_schema_doctor.py index cb0948a..7ea634c 100644 --- a/omop_alchemy/maintenance/cli_schema_doctor.py +++ b/omop_alchemy/maintenance/cli_schema_doctor.py @@ -106,8 +106,8 @@ def _build_recommendations( status=Status.WARNING, summary="Some tables were found under a different schema than expected.", action=( - "Run `omop-alchemy acknowledge-schema-migration` if this was a " - "deliberate change, or `omop-alchemy drop-orphan-schema-tables` to " + "Run `omop-config acknowledge-schema-migration` if this was a " + "deliberate change, or `omop-config drop-orphan-schema-tables` to " "clean up an orphaned copy." ), ) @@ -275,7 +275,6 @@ def collect_doctor_report( foreign_key_status = tuple( collect_foreign_key_trigger_status( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) ) @@ -299,7 +298,6 @@ def collect_doctor_report( if deep: foreign_key_validation = validate_foreign_key_constraints( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) violating_tables = sum( diff --git a/omop_alchemy/maintenance/cli_schema_info.py b/omop_alchemy/maintenance/cli_schema_info.py index 722e628..5f8c52a 100644 --- a/omop_alchemy/maintenance/cli_schema_info.py +++ b/omop_alchemy/maintenance/cli_schema_info.py @@ -374,7 +374,6 @@ def collect_maintenance_info( connection_ready = True missing_tables = collect_missing_tables( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) missing_table_count = len(missing_tables) diff --git a/omop_alchemy/maintenance/cli_schema_reconcile.py b/omop_alchemy/maintenance/cli_schema_reconcile.py index f69bef6..1069ca8 100644 --- a/omop_alchemy/maintenance/cli_schema_reconcile.py +++ b/omop_alchemy/maintenance/cli_schema_reconcile.py @@ -485,6 +485,7 @@ def reconcile_schema( actual_cluster = _backend.get_clustered_index_name( connection, maintenance_table.table_name, + role=table_role, ) if expected_cluster != actual_cluster: # May be a rename, not drift, so treat like a renamed index. diff --git a/omop_alchemy/maintenance/cli_schema_summary.py b/omop_alchemy/maintenance/cli_schema_summary.py index 58c71ea..a348de9 100644 --- a/omop_alchemy/maintenance/cli_schema_summary.py +++ b/omop_alchemy/maintenance/cli_schema_summary.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from oa_configurator import qualified +from oa_configurator import Role, qualified, schema_of from .tables import TableCategory, select_omop_tables @@ -16,6 +16,7 @@ class TableSummaryResult: table_name: str category: TableCategory + role: Role model_name: str primary_key_columns: tuple[str, ...] exists: bool @@ -25,7 +26,6 @@ class TableSummaryResult: def collect_data_summary( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, existing_only: bool = True, ) -> list[TableSummaryResult]: @@ -36,7 +36,7 @@ def collect_data_summary( results: list[TableSummaryResult] = [] with engine.connect() as connection: for table in tables: - exists = inspector.has_table(table.table_name, schema=db_schema) + exists = inspector.has_table(table.table_name, schema=schema_of(engine, role=table.role)) if not exists and existing_only: continue @@ -45,7 +45,7 @@ def collect_data_summary( row_count = int( connection.execute( sa.text( - f"SELECT COUNT(*) FROM {qualified(connection, table.table_name)}" + f"SELECT COUNT(*) FROM {qualified(connection, table.table_name, role=table.role)}" ) ).scalar_one() ) @@ -54,6 +54,7 @@ def collect_data_summary( TableSummaryResult( table_name=table.table_name, category=table.category, + role=table.role, model_name=table.model_name, primary_key_columns=table.primary_key_names, exists=exists, diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index ad56232..92e4bc2 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance +from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance, schema_of from orm_loader.helpers import Base from ._cli_utils import Status, dry_label, dry_status from .tables import ( @@ -42,14 +42,11 @@ def _table_dependencies(table: MaintenanceTable) -> tuple[str, ...]: def collect_missing_tables( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = True, ) -> list[MaintenanceTable]: - """Return ORM-managed tables that are absent from the target database.""" - inspector = sa.inspect(engine) + """Return ORM-managed tables that are absent from the target database, each checked against its own role's schema.""" return missing_maintenance_tables( - inspector, - db_schema=db_schema, + engine, vocabulary_included=vocabulary_included, ) @@ -58,7 +55,6 @@ def create_missing_tables( engine: sa.Engine, *, vocab_engine: sa.Engine | None = None, - db_schema: str | None = None, vocabulary_included: bool = True, dry_run: bool = False, resolved: ResolvedCDMDatabase | None = None, @@ -80,14 +76,30 @@ def create_missing_tables( """ vocab_engine = vocab_engine if vocab_engine is not None else engine if not dry_run: - ensure_schema(engine, db_schema) + ensure_schema(engine, schema_of(engine, role=Role.PRIMARY)) + # Primary alone isn't enough: on a genuinely fresh database, + # vocab_schema/results_schema don't exist yet either -- without this, + # create_all() below fails "schema does not exist" for every + # vocab/results table instead of creating it. ensure_schema() itself + # already no-ops for None/already-default/schema-incapable dialects, + # so calling it for RESULTS (which usually equals the primary schema) + # is safe. + if resolved is not None: + ensure_schema(engine, resolved.schema_for_role(Role.RESULTS)) + ensure_schema(vocab_engine, resolved.schema_for_role(Role.VOCAB)) inspector = sa.inspect(engine) missing_tables = collect_missing_tables( engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) - existing_table_names = set(inspector.get_table_names(schema=db_schema)) + # Each role's own schema, not just primary: a dependency table can be + # vocab- or results-role, and checking only the primary schema here used + # to make every already-existing vocab/results table look absent to the + # dependency-resolution pass below, incorrectly blocking creation of + # clinical tables that reference them. + existing_table_names: set[str] = set() + for role in Role: + existing_table_names |= set(inspector.get_table_names(schema=schema_of(engine, role=role))) missing_table_names = {table.table_name for table in missing_tables} blocked_dependencies: dict[str, tuple[str, ...]] = {} diff --git a/omop_alchemy/maintenance/cli_tables.py b/omop_alchemy/maintenance/cli_tables.py index e04dcf3..f4aa752 100644 --- a/omop_alchemy/maintenance/cli_tables.py +++ b/omop_alchemy/maintenance/cli_tables.py @@ -7,7 +7,7 @@ import sqlalchemy as sa import typer -from oa_configurator import autocommit_connection, qualified +from oa_configurator import Role, autocommit_connection, qualified, schema_of from ..backends import resolve_backend, require_backend_support, backend_support_note from ._cli_utils import Status, dry_label, dry_status, omop_command, resolve_selection from .tables import ( @@ -39,6 +39,7 @@ class AnalyzeTableResult: table_name: str category: TableCategory + role: Role operation: str status: Status detail: str @@ -47,7 +48,6 @@ class AnalyzeTableResult: def analyze_tables( engine: sa.Engine, *, - db_schema: str | None = None, scope: TableCategory | None = None, table_names: tuple[str, ...] | None = None, vacuum: bool = False, @@ -70,11 +70,13 @@ def analyze_tables( with connection_factory as connection: for maintenance_table in selected_tables: - if not inspector.has_table(maintenance_table.table_name, schema=db_schema): + table_schema = schema_of(engine, role=maintenance_table.role) + if not inspector.has_table(maintenance_table.table_name, schema=table_schema): results.append( AnalyzeTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, + role=maintenance_table.role, operation=operation, status=Status.SKIPPED, detail="table not present in target database", @@ -83,12 +85,15 @@ def analyze_tables( continue if not dry_run: - backend.analyze_table(connection, maintenance_table.table_name, vacuum=vacuum) + backend.analyze_table( + connection, maintenance_table.table_name, vacuum=vacuum, role=maintenance_table.role + ) results.append( AnalyzeTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, + role=maintenance_table.role, operation=operation, status=dry_status(dry_run), detail=dry_label(dry_run, f"{operation.lower()} would run", f"{operation.lower()} completed"), @@ -108,29 +113,37 @@ class TruncateTableResult: table_name: str category: TableCategory + role: Role row_count: int | None status: Status detail: str def _blocking_foreign_key_references( + engine: sa.Engine, inspector: sa.Inspector, *, - db_schema: str | None, selected_table_names: set[str], ) -> dict[str, set[str]]: - """Return tables outside the selection that FK-reference at least one selected table, preventing truncation.""" - blockers: dict[str, set[str]] = {} + """Return tables outside the selection that FK-reference at least one selected table, preventing truncation. - for table_name in inspector.get_table_names(schema=db_schema): - if table_name in selected_table_names: - continue + A blocking table can live under any role's schema (a vocab table can + FK-reference a clinical table's PK, or vice versa), so every role's + schema is scanned, not just the primary one. + """ + blockers: dict[str, set[str]] = {} - for foreign_key in inspector.get_foreign_keys(table_name, schema=db_schema): - referred_table = foreign_key.get("referred_table") - if referred_table not in selected_table_names: + for role in Role: + role_schema = schema_of(engine, role=role) + for table_name in inspector.get_table_names(schema=role_schema): + if table_name in selected_table_names: continue - blockers.setdefault(str(referred_table), set()).add(table_name) + + for foreign_key in inspector.get_foreign_keys(table_name, schema=role_schema): + referred_table = foreign_key.get("referred_table") + if referred_table not in selected_table_names: + continue + blockers.setdefault(str(referred_table), set()).add(table_name) return blockers @@ -155,7 +168,6 @@ def _format_blocking_reference_error(blockers: dict[str, set[str]]) -> str: def truncate_tables( engine: sa.Engine, *, - db_schema: str | None = None, scope: TableCategory | None = None, table_names: tuple[str, ...] | None = None, restart_identities: bool = False, @@ -174,14 +186,18 @@ def truncate_tables( inspector = sa.inspect(engine) results: list[TruncateTableResult] = [] existing_tables: list[str] = [] + existing_table_names_by_role: dict[Role, list[str]] = {} with engine.begin() as connection: for maintenance_table in selected_tables: - if not inspector.has_table(maintenance_table.table_name, schema=db_schema): + if not inspector.has_table( + maintenance_table.table_name, schema=schema_of(engine, role=maintenance_table.role) + ): results.append( TruncateTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, + role=maintenance_table.role, row_count=None, status=Status.SKIPPED, detail="table not present in target database", @@ -191,14 +207,18 @@ def truncate_tables( row_count = int( connection.exec_driver_sql( - f"SELECT COUNT(*) FROM {qualified(connection, maintenance_table.table_name)}" + f"SELECT COUNT(*) FROM {qualified(connection, maintenance_table.table_name, role=maintenance_table.role)}" ).scalar_one() ) existing_tables.append(maintenance_table.table_name) + existing_table_names_by_role.setdefault(maintenance_table.role, []).append( + maintenance_table.table_name + ) results.append( TruncateTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, + role=maintenance_table.role, row_count=row_count, status=dry_status(dry_run), detail=dry_label(dry_run, "table would be truncated", "table truncated"), @@ -207,20 +227,27 @@ def truncate_tables( if existing_tables and not dry_run and not cascade: blockers = _blocking_foreign_key_references( + engine, inspector, - db_schema=db_schema, selected_table_names=set(existing_tables), ) if blockers: raise RuntimeError(_format_blocking_reference_error(blockers)) if existing_tables and not dry_run: - backend.truncate_table_batch( - connection, - existing_tables, - restart_identities=restart_identities, - cascade=cascade, - ) + # One TRUNCATE per role: truncate_table_batch qualifies every name + # in its list with a single role, so a selection spanning more + # than one role (category and role are independent axes -- see + # MaintenanceTable.role) is split into one batch per role rather + # than misqualifying some of the names. + for role, table_names_for_role in existing_table_names_by_role.items(): + backend.truncate_table_batch( + connection, + table_names_for_role, + restart_identities=restart_identities, + cascade=cascade, + role=role, + ) return results @@ -235,6 +262,7 @@ class SequenceTarget: table_name: str category: TableCategory + role: Role pk_column_name: str @@ -244,6 +272,7 @@ class SequenceResetResult: table_name: str category: TableCategory + role: Role pk_column_name: str sequence_name: str | None next_value: int | None @@ -268,6 +297,7 @@ def collect_sequence_targets( SequenceTarget( table_name=table.table_name, category=table.category, + role=table.role, pk_column_name=pk_column_name, ) ) @@ -277,7 +307,6 @@ def collect_sequence_targets( def reset_model_sequences( engine: sa.Engine, *, - db_schema: str | None = None, vocabulary_included: bool = False, dry_run: bool = False, ) -> list[SequenceResetResult]: @@ -290,11 +319,11 @@ def reset_model_sequences( with engine.begin() as connection: for target in targets: - if not inspector.has_table(target.table_name, schema=db_schema): + if not inspector.has_table(target.table_name, schema=schema_of(engine, role=target.role)): continue sequence_name = backend.find_sequence_name( - connection, target.table_name, target.pk_column_name + connection, target.table_name, target.pk_column_name, role=target.role ) if sequence_name is None: @@ -302,6 +331,7 @@ def reset_model_sequences( SequenceResetResult( table_name=target.table_name, category=target.category, + role=target.role, pk_column_name=target.pk_column_name, sequence_name=None, next_value=None, @@ -311,7 +341,7 @@ def reset_model_sequences( ) continue - fully_qualified = qualified(connection, target.table_name) + fully_qualified = qualified(connection, target.table_name, role=target.role) current_max = connection.execute( sa.text( f"SELECT COALESCE(MAX({target.pk_column_name}), 0) " @@ -327,6 +357,7 @@ def reset_model_sequences( SequenceResetResult( table_name=target.table_name, category=target.category, + role=target.role, pk_column_name=target.pk_column_name, sequence_name=sequence_name, next_value=next_value, @@ -372,7 +403,6 @@ def analyze_tables_command( with console.status("Refreshing planner statistics for selected tables..."): results = analyze_tables( engine, - db_schema=conn.resolved.schema_name, scope=resolved_scope, table_names=resolved_tables, vacuum=vacuum, @@ -402,7 +432,6 @@ def reset_sequences_command( with console.status("Resetting PostgreSQL sequences..."): results = reset_model_sequences( engine, - db_schema=conn.resolved.schema_name, vocabulary_included=vocabulary_included, dry_run=dry_run, ) @@ -461,7 +490,6 @@ def truncate_tables_command( with console.status("Truncating selected tables..."): results = truncate_tables( engine, - db_schema=conn.resolved.schema_name, scope=resolved_scope, table_names=resolved_tables, restart_identities=restart_identities, diff --git a/omop_alchemy/maintenance/cli_vocab.py b/omop_alchemy/maintenance/cli_vocab.py index 9458535..f524646 100644 --- a/omop_alchemy/maintenance/cli_vocab.py +++ b/omop_alchemy/maintenance/cli_vocab.py @@ -377,7 +377,6 @@ def load_vocab_source( engine, enable=False, vocabulary_included=True, - db_schema=db_schema, dry_run=False, ) _emit( @@ -390,7 +389,6 @@ def load_vocab_source( engine, enable=False, vocabulary_included=True, - db_schema=db_schema, dry_run=False, ) index_warnings = tuple( @@ -543,7 +541,6 @@ def load_vocab_source( engine, enable=True, vocabulary_included=True, - db_schema=db_schema, dry_run=False, cluster=False, ) @@ -557,7 +554,6 @@ def load_vocab_source( engine, enable=True, vocabulary_included=True, - db_schema=db_schema, dry_run=False, ) @@ -571,7 +567,6 @@ def load_vocab_source( if not dry_run and engine.dialect.name == Dialect.POSTGRESQL: sequence_results = reset_model_sequences( engine, - db_schema=db_schema, vocabulary_included=True, dry_run=False, ) diff --git a/omop_alchemy/maintenance/tables.py b/omop_alchemy/maintenance/tables.py index 94021f7..d24023f 100644 --- a/omop_alchemy/maintenance/tables.py +++ b/omop_alchemy/maintenance/tables.py @@ -5,6 +5,8 @@ from typing import Iterable import sqlalchemy as sa +from oa_configurator import Role, schema_of +from orm_loader.helpers import role_of_table class TableCategory(StrEnum): @@ -56,6 +58,18 @@ class MaintenanceTable: table: sa.Table primary_key_columns: tuple[sa.Column[object], ...] + @property + def role(self) -> Role: + """The schema_translate_map role this table's data physically lives + under (oa_configurator.Role), read off its own declared schema tag. + + Independent of TableCategory: category is a logical/folder grouping + (e.g. cohort/cohort_definition are RESULTS-category despite + classifying as "derived" in the CDM sense), role is where the + table's rows physically live. + """ + return role_of_table(self.table) + @property def is_vocabulary(self) -> bool: return self.category is TableCategory.VOCABULARY @@ -234,30 +248,47 @@ def select_omop_tables( def existing_maintenance_tables( - inspector: sa.Inspector, + bindable: sa.Engine | sa.Connection, *, - db_schema: str | None, vocabulary_included: bool, require_single_integer_primary_key: bool = False, ) -> list[MaintenanceTable]: + """ORM-managed tables that already exist, each checked against its own role's schema. + + Parameters + ---------- + bindable : sqlalchemy.Engine or sqlalchemy.Connection + Used both to inspect the database and, via its schema_translate_map, + to resolve each table's own role to a physical schema + (``schema_of(bindable, role=table.role)``) -- a blanket schema + passed in once would silently misclassify every vocab/results + table checked against a database with a genuine primary/vocab/ + results split. + """ + inspector = sa.inspect(bindable) return [ table for table in select_omop_tables( vocabulary_included=vocabulary_included, require_single_integer_primary_key=require_single_integer_primary_key, ) - if inspector.has_table(table.table_name, schema=db_schema) + if inspector.has_table(table.table_name, schema=schema_of(bindable, role=table.role)) ] def missing_maintenance_tables( - inspector: sa.Inspector, + bindable: sa.Engine | sa.Connection, *, - db_schema: str | None, vocabulary_included: bool, ) -> list[MaintenanceTable]: + """ORM-managed tables that are absent, each checked against its own role's schema. + + See :func:`existing_maintenance_tables` for why *bindable* replaces a + single ``db_schema`` string. + """ + inspector = sa.inspect(bindable) return [ table for table in select_omop_tables(vocabulary_included=vocabulary_included) - if not inspector.has_table(table.table_name, schema=db_schema) + if not inspector.has_table(table.table_name, schema=schema_of(bindable, role=table.role)) ] diff --git a/tests/test_backends_non_default_schema_postgres.py b/tests/test_backends_non_default_schema_postgres.py index 6e5766c..299e10c 100644 --- a/tests/test_backends_non_default_schema_postgres.py +++ b/tests/test_backends_non_default_schema_postgres.py @@ -57,34 +57,34 @@ def scoped(pg_engine: sa.Engine) -> Iterator[_Scoped]: def test_fk_trigger_toggle_targets_the_configured_schema(scoped: _Scoped) -> None: - create_missing_tables(scoped.engine, db_schema=scoped.schema, vocabulary_included=True) + create_missing_tables(scoped.engine, vocabulary_included=True) - disabled = manage_foreign_key_triggers(scoped.engine, enable=False, db_schema=scoped.schema) + disabled = manage_foreign_key_triggers(scoped.engine, enable=False) assert disabled assert all(result.status == Status.APPLIED for result in disabled) status_after_disable = { result.table_name: result - for result in collect_foreign_key_trigger_status(scoped.engine, db_schema=scoped.schema) + for result in collect_foreign_key_trigger_status(scoped.engine) } person_status = status_after_disable["person"] assert person_status.enabled_trigger_count == 0 assert person_status.disabled_trigger_count > 0 - enabled = manage_foreign_key_triggers(scoped.engine, enable=True, db_schema=scoped.schema) + enabled = manage_foreign_key_triggers(scoped.engine, enable=True) assert all(result.status == Status.APPLIED for result in enabled) status_after_enable = { result.table_name: result - for result in collect_foreign_key_trigger_status(scoped.engine, db_schema=scoped.schema) + for result in collect_foreign_key_trigger_status(scoped.engine) } assert status_after_enable["person"].disabled_trigger_count == 0 def test_index_disable_and_enable_targets_the_configured_schema(scoped: _Scoped) -> None: - create_missing_tables(scoped.engine, db_schema=scoped.schema, vocabulary_included=True) + create_missing_tables(scoped.engine, vocabulary_included=True) - disabled = manage_indexes(scoped.engine, enable=False, db_schema=scoped.schema) + disabled = manage_indexes(scoped.engine, enable=False) assert disabled assert all(result.status in (Status.APPLIED, Status.SKIPPED) for result in disabled) @@ -93,7 +93,7 @@ def test_index_disable_and_enable_targets_the_configured_schema(scoped: _Scoped) idx["name"] for idx in inspector.get_indexes("person", schema=scoped.schema) } - enabled = manage_indexes(scoped.engine, enable=True, db_schema=scoped.schema) + enabled = manage_indexes(scoped.engine, enable=True) assert all(result.status in (Status.APPLIED, Status.SKIPPED) for result in enabled) inspector = sa.inspect(scoped.engine) @@ -104,9 +104,9 @@ def test_index_disable_and_enable_targets_the_configured_schema(scoped: _Scoped) def test_fulltext_install_targets_the_configured_schema(scoped: _Scoped) -> None: - create_missing_tables(scoped.engine, db_schema=scoped.schema, vocabulary_included=True) + create_missing_tables(scoped.engine, vocabulary_included=True) - results = install_fulltext_columns(scoped.engine, db_schema=scoped.schema) + results = install_fulltext_columns(scoped.engine) assert results assert all(result.status == Status.APPLIED for result in results) @@ -119,10 +119,10 @@ def test_fulltext_install_targets_the_configured_schema(scoped: _Scoped) -> None def test_sequence_reset_targets_the_configured_schema(scoped: _Scoped) -> None: - create_missing_tables(scoped.engine, db_schema=scoped.schema, vocabulary_included=True) + create_missing_tables(scoped.engine, vocabulary_included=True) results = { - r.table_name: r for r in reset_model_sequences(scoped.engine, db_schema=scoped.schema) + r.table_name: r for r in reset_model_sequences(scoped.engine) } person_result = results["person"] diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 8797792..1ae731f 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -11,7 +11,7 @@ collect_foreign_key_trigger_status, manage_foreign_key_triggers, ) -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Role, StackConfig runner = CliRunner() @@ -100,10 +100,10 @@ def name(self) -> str: def dialect(self) -> str: return "postgresql" - def analyze_table(self, conn, table_name, *, vacuum=False) -> None: + def analyze_table(self, conn, table_name, *, vacuum=False, role=None) -> None: pass - def toggle_fk_triggers(self, conn, table_name, *, enable: bool) -> None: + def toggle_fk_triggers(self, conn, table_name, *, enable: bool, role=None) -> None: action = "ENABLE" if enable else "DISABLE" conn.exec_driver_sql(f"ALTER TABLE {table_name} {action} TRIGGER ALL") @@ -138,10 +138,11 @@ def begin(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_fk_info", - lambda engine, *, db_schema=None, vocabulary_included=False: [ + lambda engine, *, vocabulary_included=False: [ type("Target", (), { "table_name": "person", "category": "clinical", + "role": Role.PRIMARY, "model_name": "Person", "model_module": "omop_alchemy.cdm.model.clinical.person", "outgoing_constraint_count": 1, @@ -150,6 +151,7 @@ def begin(self): type("Target", (), { "table_name": "visit_occurrence", "category": "health_system", + "role": Role.PRIMARY, "model_name": "VisitOccurrence", "model_module": "omop_alchemy.cdm.model.health_system.visit_occurrence", "outgoing_constraint_count": 2, @@ -159,7 +161,7 @@ def begin(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_strict_validation_failures", - lambda connection, backend, *, db_schema=None, vocabulary_included=False: { + lambda connection, backend, *, vocabulary_included=False: { "visit_occurrence": [ ForeignKeyConstraintViolation( source_table_name="visit_occurrence", @@ -208,10 +210,11 @@ def begin(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_fk_info", - lambda engine, *, db_schema=None, vocabulary_included=False: [ + lambda engine, *, vocabulary_included=False: [ type("Target", (), { "table_name": "person", "category": "clinical", + "role": Role.PRIMARY, "model_name": "Person", "model_module": "omop_alchemy.cdm.model.clinical.person", "outgoing_constraint_count": 1, @@ -221,7 +224,7 @@ def begin(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_strict_validation_failures", - lambda connection, backend, *, db_schema=None, vocabulary_included=False: {}, + lambda connection, backend, *, vocabulary_included=False: {}, ) results = manage_foreign_key_triggers( @@ -305,10 +308,11 @@ def connect(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_fk_info", - lambda engine, *, db_schema=None, vocabulary_included=False: [ + lambda engine, *, vocabulary_included=False: [ type("Target", (), { "table_name": "person", "category": "clinical", + "role": Role.PRIMARY, "model_name": "Person", "model_module": "omop_alchemy.cdm.model.clinical.person", "outgoing_constraint_count": 1, @@ -317,6 +321,7 @@ def connect(self): type("Target", (), { "table_name": "visit_occurrence", "category": "health_system", + "role": Role.PRIMARY, "model_name": "VisitOccurrence", "model_module": "omop_alchemy.cdm.model.health_system.visit_occurrence", "outgoing_constraint_count": 2, @@ -326,7 +331,7 @@ def connect(self): ) monkeypatch.setattr( "omop_alchemy.maintenance.cli_foreign_keys._collect_strict_validation_failures", - lambda connection, backend, *, db_schema=None, vocabulary_included=False: { + lambda connection, backend, *, vocabulary_included=False: { "visit_occurrence": [ ForeignKeyConstraintViolation( source_table_name="visit_occurrence", @@ -386,6 +391,7 @@ def fake_validate_foreign_key_constraints( ForeignKeyValidationResult( table_name="visit_occurrence", category=TableCategory.HEALTH_SYSTEM, + role=Role.PRIMARY, outgoing_constraint_count=2, incoming_constraint_count=0, violating_constraint_count=1, diff --git a/tests/test_fulltext.py b/tests/test_fulltext.py index 9fc4c34..41e84e7 100644 --- a/tests/test_fulltext.py +++ b/tests/test_fulltext.py @@ -44,7 +44,13 @@ def __init__(self, *, rowcount: int = 7, db_schema: str | None = "public"): self._db_schema = db_schema def get_execution_options(self) -> dict[str, object]: - return {SCHEMA_TRANSLATE_MAP_KEY: {SchemaRole.PRIMARY.value: self._db_schema}} + return { + SCHEMA_TRANSLATE_MAP_KEY: { + SchemaRole.PRIMARY.value: self._db_schema, + SchemaRole.VOCAB.value: self._db_schema, + SchemaRole.RESULTS.value: self._db_schema, + } + } def exec_driver_sql( self, @@ -120,7 +126,6 @@ def test_install_fulltext_columns_builds_postgresql_ddl_and_registers_metadata() results = install_fulltext_columns( engine, # type: ignore[arg-type] - db_schema="public", create_indexes=True, fastupdate=True, ) @@ -147,7 +152,6 @@ def test_populate_fulltext_columns_issues_update_with_regconfig_and_row_counts() results = populate_fulltext_columns( engine, # type: ignore[arg-type] - db_schema="public", regconfig="simple", ) @@ -167,7 +171,6 @@ def test_drop_fulltext_columns_drops_schema_objects_and_unregisters_metadata(): results = drop_fulltext_columns( engine, # type: ignore[arg-type] - db_schema="public", drop_indexes=True, ) @@ -226,13 +229,11 @@ def test_fulltext_install_cli_passes_options(monkeypatch): def fake_install_fulltext_columns( engine: object, *, - db_schema: str | None = None, create_indexes: bool = True, fastupdate: bool = False, dry_run: bool = False, ): calls["engine"] = engine - calls["db_schema"] = db_schema calls["create_indexes"] = create_indexes calls["fastupdate"] = fastupdate calls["dry_run"] = dry_run @@ -265,7 +266,6 @@ def fake_install_fulltext_columns( ) assert result.exit_code == 0 - assert calls["db_schema"] == "public" assert calls["fastupdate"] is True assert calls["dry_run"] is True assert "fulltext install" in result.stdout diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 93cab93..db9ec17 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -2,7 +2,7 @@ import sqlalchemy as sa from pydantic import ValidationError from typer.testing import CliRunner -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig, qualified +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Role, StackConfig, qualified from oa_configurator.testing import DIALECT_PARAMS from omop_alchemy.backends.sqlite import SQLiteBackend @@ -201,9 +201,9 @@ def test_manage_indexes_enable_analyzes_tables_with_new_indexes(sqlite_indexed_e analyzed_tables: list[str] = [] original_analyze = SQLiteBackend.analyze_table - def recording_analyze(self, conn, table_name, *, vacuum=False): + def recording_analyze(self, conn, table_name, *, vacuum=False, role=Role.PRIMARY): analyzed_tables.append(table_name) - return original_analyze(self, conn, table_name, vacuum=vacuum) + return original_analyze(self, conn, table_name, vacuum=vacuum, role=role) monkeypatch.setattr(SQLiteBackend, "analyze_table", recording_analyze) @@ -345,10 +345,10 @@ def test_manage_indexes_enable_clusters_then_analyzes(sqlite_indexed_engine, mon calls: list[str] = [] - def fake_cluster_table(self, conn, table_name, index_name): + def fake_cluster_table(self, conn, table_name, index_name, *, role=Role.PRIMARY): calls.append(f"cluster:{table_name}") - def fake_analyze_table(self, conn, table_name, *, vacuum=False): + def fake_analyze_table(self, conn, table_name, *, vacuum=False, role=Role.PRIMARY): calls.append(f"analyze:{table_name}") monkeypatch.setattr(SQLiteBackend, "cluster_table", fake_cluster_table) @@ -395,6 +395,7 @@ def fake_manage_indexes( operation="index", table_name="person", category=TableCategory.CLINICAL, + role=Role.PRIMARY, index_name=PERSON_GENDER_INDEX, column_names=("gender_concept_id",), unique=False, @@ -459,6 +460,7 @@ def fake_manage_indexes( operation="index", table_name="person", category=TableCategory.CLINICAL, + role=Role.PRIMARY, index_name=PERSON_GENDER_INDEX, column_names=("gender_concept_id",), unique=False, @@ -731,14 +733,14 @@ def test_manage_indexes_enable_cluster_uses_restored_physical_name(sqlite_indexe calls: list[tuple[str, str]] = [] - def fake_cluster_table(self, conn, table_name, index_name): + def fake_cluster_table(self, conn, table_name, index_name, *, role=Role.PRIMARY): calls.append((table_name, index_name)) monkeypatch.setattr(SQLiteBackend, "cluster_table", fake_cluster_table) monkeypatch.setattr( SQLiteBackend, "analyze_table", - lambda self, conn, table_name, *, vacuum=False: None, + lambda self, conn, table_name, *, vacuum=False, role=Role.PRIMARY: None, ) manage_indexes(engine, enable=True, cluster=True) @@ -815,6 +817,7 @@ def _warning_result() -> IndexManagementResult: operation="index", table_name="person", category=TableCategory.CLINICAL, + role=Role.PRIMARY, index_name="idx_gender_partial", column_names=("gender_concept_id",), unique=False, @@ -836,6 +839,7 @@ def test_render_index_summary_omits_warnings_row_when_none(): operation="index", table_name="person", category=TableCategory.CLINICAL, + role=Role.PRIMARY, index_name=PERSON_GENDER_INDEX, column_names=("gender_concept_id",), unique=False, diff --git a/tests/test_inspect_functional_index_probe.py b/tests/test_inspect_functional_index_probe.py new file mode 100644 index 0000000..81055e7 --- /dev/null +++ b/tests/test_inspect_functional_index_probe.py @@ -0,0 +1,25 @@ +import pytest +import sqlalchemy as sa +from oa_configurator.testing import isolated_test_schema +from omop_alchemy.maintenance.cli_schema import create_missing_tables + +pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] + +def test_inspect_functional_index(pg_db, pg_engine): + with isolated_test_schema(pg_engine, prefix="funcidx_probe") as schema: + import dataclasses + resolved = dataclasses.replace( + pg_db.resolved, schema_name=schema, vocab_schema=schema, results_schema=schema + ) + engine = pg_engine.execution_options( + schema_translate_map={"primary": schema, "vocab": schema, "results": schema} + ) + create_missing_tables(engine, vocabulary_included=True, resolved=resolved) + inspector = sa.inspect(engine) + with engine.connect() as conn: + inspector = sa.inspect(conn) + for idx in inspector.get_indexes("concept", schema=schema): + if idx["name"] == "ix_concept_concept_name_lower": + print("FULL DICT:", idx) + for k, v in idx.items(): + print(f" {k!r}: {v!r}") diff --git a/tests/test_load_vocab_source.py b/tests/test_load_vocab_source.py index 6975087..c4cdb11 100644 --- a/tests/test_load_vocab_source.py +++ b/tests/test_load_vocab_source.py @@ -2,7 +2,7 @@ import pytest import sqlalchemy as sa -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Role, StackConfig from sqlalchemy.orm import sessionmaker from typer.testing import CliRunner @@ -698,6 +698,7 @@ def fake_manage_indexes(engine, *, enable, **kwargs): operation="index", table_name="concept", category=TableCategory.VOCABULARY, + role=Role.VOCAB, index_name="idx_concept_partial", column_names=("domain_id",), unique=False, @@ -713,6 +714,7 @@ def fake_manage_indexes(engine, *, enable, **kwargs): operation="index", table_name="concept", category=TableCategory.VOCABULARY, + role=Role.VOCAB, index_name="ix_concept_domain_id", column_names=("domain_id",), unique=False, diff --git a/tests/test_schema_doctor.py b/tests/test_schema_doctor.py index 9b76c54..8168c08 100644 --- a/tests/test_schema_doctor.py +++ b/tests/test_schema_doctor.py @@ -19,12 +19,10 @@ def fail_config_resolution(): def collect_missing( supplied_engine, *, - db_schema=None, vocabulary_included=True, ): inspected.update( engine=supplied_engine, - db_schema=db_schema, vocabulary_included=vocabulary_included, ) return [] @@ -51,7 +49,6 @@ def track_dispose(self, *args, **kwargs): assert report.info.connection_ready is True assert inspected == { "engine": engine, - "db_schema": "analytics", "vocabulary_included": False, } assert engine not in disposed_engines diff --git a/tests/test_schema_provenance_guard.py b/tests/test_schema_provenance_guard.py index e2f9688..3e9120e 100644 --- a/tests/test_schema_provenance_guard.py +++ b/tests/test_schema_provenance_guard.py @@ -62,7 +62,6 @@ def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_db, pg_engi ) create_missing_tables( engine_a, - db_schema=schema_a, resolved=_resolved(pg_db, database_name=database_name, schema=schema_a), ) @@ -72,7 +71,6 @@ def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_db, pg_engi with pytest.raises(SchemaDriftError): create_missing_tables( engine_b, - db_schema=schema_b, resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), ) diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index 9f63084..11a0273 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -75,7 +75,7 @@ def reconcile_engine(request) -> _ReconcileEngine: resolved = _sqlite_resolved(engine) create_missing_tables(engine) if request.param == Dialect.POSTGRESQL: - manage_indexes(engine, enable=True, db_schema=schema) + manage_indexes(engine, enable=True) else: manage_indexes(engine, enable=True) return _ReconcileEngine(engine, resolved) @@ -167,7 +167,7 @@ def test_reconcile_schema_reports_relocated_when_table_found_in_another_schema(p # vocabulary_included defaults to True: person's gender_concept_id FK # targets a vocab table, so excluding vocab here would leave that FK # unresolved and person itself blocked from creation. - create_missing_tables(engine, db_schema=schema_a, resolved=resolved) + create_missing_tables(engine, resolved=resolved) with engine.begin() as connection: connection.exec_driver_sql(f'ALTER TABLE "{schema_a}".person SET SCHEMA "{schema_b}"') @@ -211,16 +211,23 @@ def test_reconcile_schema_with_resolved_qualifies_each_table_to_its_own_role_sch Role.PRIMARY.value: primary_schema, "vocab": vocab_schema, "results": results_schema } ) - create_missing_tables(engine, db_schema=primary_schema, resolved=resolved) + create_missing_tables(engine, resolved=resolved) + manage_indexes(engine, enable=True, vocabulary_included=True) report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) - # cluster/index issues are excluded here. manage_indexes()/ - # cli_indexes.py's cluster commands have their own, separate - # cross-schema bug (found while writing this test, not yet - # investigated). This test only verifies that reconcile_schema - # resolves each table's own role schema instead of one blanket value. - checked_components = {"table", "column", "primary_key", "foreign_key"} + # index issues are excluded here: ix_concept_concept_name_lower is a + # functional index (lower(concept_name)), and reconcile_schema's + # index diff can't read its expression column back from the + # inspector. Confirmed pre-existing and schema-independent (it + # reproduces against a single non-default schema too), so it is out + # of scope for this test, which only covers cross-schema behaviour. + # cluster issues are no longer excluded: reconcile_schema's cluster + # check used to call get_clustered_index_name() without a role, + # always inspecting the primary schema, so a vocab/results table's + # real cluster state was invisible whenever its schema differed from + # primary. Fixed by passing role=table_role through. + checked_components = {"table", "column", "primary_key", "foreign_key", "cluster"} for table_name in ("person", "concept", "observation_period"): issues = [ issue @@ -261,7 +268,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_inde monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name: ( + lambda self, conn, table_name, role=None: ( "idx_episode_person" if table_name == "episode" else None ), ) @@ -288,7 +295,7 @@ def test_reconcile_schema_cluster_check_still_reports_real_mismatch(fresh_reconc monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name: ( + lambda self, conn, table_name, role=None: ( "some_unrelated_index" if table_name == "episode" else None ), ) @@ -322,7 +329,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_tar monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name: ( + lambda self, conn, table_name, role=None: ( "idx_person_id" if table_name == "person" else None ), ) diff --git a/tests/test_truncate_tables.py b/tests/test_truncate_tables.py index 52eb151..0f97470 100644 --- a/tests/test_truncate_tables.py +++ b/tests/test_truncate_tables.py @@ -1,7 +1,7 @@ import importlib import pytest from typer.testing import CliRunner -from oa_configurator import CDMDatabaseConfig, ConnectionConfig, StackConfig +from oa_configurator import CDMDatabaseConfig, ConnectionConfig, Role, StackConfig from omop_alchemy.maintenance.cli import app from omop_alchemy.maintenance._cli_utils import Status @@ -73,7 +73,6 @@ def test_truncate_tables_cli_invokes_management(monkeypatch): def fake_truncate_tables( engine: object, *, - db_schema: str | None = None, scope: TableCategory | None = None, table_names: tuple[str, ...] | None = None, restart_identities: bool = False, @@ -81,7 +80,6 @@ def fake_truncate_tables( dry_run: bool = False, ) -> list[TruncateTableResult]: calls["engine"] = engine - calls["db_schema"] = db_schema calls["scope"] = scope calls["table_names"] = table_names calls["restart_identities"] = restart_identities @@ -91,6 +89,7 @@ def fake_truncate_tables( TruncateTableResult( table_name="person", category=TableCategory.CLINICAL, + role=Role.PRIMARY, row_count=10, status=Status.PLANNED, detail="table would be truncated", diff --git a/tests/test_vocab_results_role_wiring_postgres.py b/tests/test_vocab_results_role_wiring_postgres.py index 1392862..3eb15bc 100644 --- a/tests/test_vocab_results_role_wiring_postgres.py +++ b/tests/test_vocab_results_role_wiring_postgres.py @@ -15,6 +15,7 @@ from __future__ import annotations +import uuid from contextlib import ExitStack from datetime import date from typing import Iterator, NamedTuple @@ -23,7 +24,7 @@ import sqlalchemy as sa import sqlalchemy.orm as so -from oa_configurator import Role +from oa_configurator import ResolvedCDMDatabase, ResolvedConnection, Role from oa_configurator.testing import isolated_test_schema from omop_alchemy.cdm.model.clinical import Observation, Person @@ -116,7 +117,7 @@ def _bootstrap_vocab(engine: sa.Engine, vocab_schema: str) -> None: def test_tables_land_in_the_schema_their_role_declares(three_schema: _ThreeSchema) -> None: create_missing_tables( - three_schema.engine, db_schema=three_schema.clinical_schema, vocabulary_included=True + three_schema.engine, vocabulary_included=True ) inspector = sa.inspect(three_schema.engine) @@ -136,7 +137,7 @@ def test_clinical_to_vocab_join_compiles_and_executes_in_one_query( three_schema: _ThreeSchema, ) -> None: create_missing_tables( - three_schema.engine, db_schema=three_schema.clinical_schema, vocabulary_included=True + three_schema.engine, vocabulary_included=True ) _bootstrap_vocab(three_schema.engine, three_schema.vocab_schema) @@ -183,3 +184,60 @@ def test_clinical_to_vocab_join_compiles_and_executes_in_one_query( sa.select(Cohort.cohort_definition_id).where(Cohort.subject_id == 1) ).one() assert cohort_row.cohort_definition_id == 1 + + +def test_create_missing_tables_creates_vocab_and_results_schemas_on_a_fresh_database( + pg_engine: sa.Engine, cleanup_after_test +) -> None: + """create_missing_tables() used to call ensure_schema() only for the + primary schema, so a genuinely fresh database (where vocab/results + schemas don't exist yet either, unlike three_schema's fixture which + pre-creates all three) failed "schema does not exist" for every + vocab/results table. Deliberately doesn't use isolated_test_schema(): + that physically creates the schema up front, which is exactly the step + under test here. + """ + clinical_schema = f"phase32_fresh_clinical_{uuid.uuid4().hex[:8]}" + vocab_schema = f"phase32_fresh_vocab_{uuid.uuid4().hex[:8]}" + results_schema = f"phase32_fresh_results_{uuid.uuid4().hex[:8]}" + + def _drop_schemas() -> None: + with pg_engine.begin() as conn: + for schema in (clinical_schema, vocab_schema, results_schema): + conn.execute(sa.text(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE')) + + cleanup_after_test(_drop_schemas) + + url = pg_engine.url + connection = ResolvedConnection( + name="fresh_schema_test", + url=url.render_as_string(hide_password=False), + safe_url=url.render_as_string(hide_password=True), + _engine_url=url, + ) + resolved = ResolvedCDMDatabase( + name="fresh_schema_test", + connection=connection, + schema_name=clinical_schema, + vocab_connection=connection, + vocab_schema=vocab_schema, + results_schema=results_schema, + ) + engine = pg_engine.execution_options( + schema_translate_map={ + Role.PRIMARY.value: clinical_schema, + "vocab": vocab_schema, + "results": results_schema, + } + ) + + create_missing_tables( + engine, + vocabulary_included=True, + resolved=resolved, + ) + + inspector = sa.inspect(engine) + assert inspector.has_table("person", schema=clinical_schema) + assert inspector.has_table("concept", schema=vocab_schema) + assert inspector.has_table("cohort", schema=results_schema) From 99117fef7601cbe3ff8659edd4789680a0a3f2d7 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 01:15:31 +0000 Subject: [PATCH 17/32] Recitfy inline comment --- omop_alchemy/cdm/model/vocabulary/concept.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/omop_alchemy/cdm/model/vocabulary/concept.py b/omop_alchemy/cdm/model/vocabulary/concept.py index 6056f0b..c0cb629 100644 --- a/omop_alchemy/cdm/model/vocabulary/concept.py +++ b/omop_alchemy/cdm/model/vocabulary/concept.py @@ -163,8 +163,6 @@ class ConceptView(Concept, ConceptContext): Avoid in tight loops or ETL paths. """ __tablename__ = "concept" - # Must match Concept.__table_args__'s schema exactly: same (schema, name) - # key is what makes SQLAlchemy reuse Concept's own Table object here - # instead of building a second, distinct one with no FK link between them. + # Must match Concept's schema, or SQLAlchemy silently builds a second, unlinked Table object. __table_args__ = {"schema": Role.VOCAB.value} __mapper_args__ = {"concrete": False} From f4935935d61bb89623f048e3e5b0329762c638f1 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 02:30:06 +0000 Subject: [PATCH 18/32] Properly handle indices and schemas for casted indices e.g. lower() --- omop_alchemy/backends/base.py | 15 +++ omop_alchemy/backends/postgres.py | 28 ++++++ omop_alchemy/backends/resolve.py | 2 +- .../maintenance/cli_schema_reconcile.py | 94 +++++++++++++------ tests/test_schema_reconcile.py | 74 ++++++++++++--- 5 files changed, 173 insertions(+), 40 deletions(-) diff --git a/omop_alchemy/backends/base.py b/omop_alchemy/backends/base.py index 38e68cb..937ac7d 100644 --- a/omop_alchemy/backends/base.py +++ b/omop_alchemy/backends/base.py @@ -134,6 +134,21 @@ def get_clustered_index_name( ) -> str | None: raise FeatureNotSupportedError("Cluster index inspection", self) + # ── Schema reconciliation ──────────────────────────────────────────────── + + def normalize_index_expression(self, sql_text: str) -> str: + """Canonicalize a reflected functional-index expression for comparison + against the ORM's own compiled expression text. + + A backend's own catalog reflection can introduce dialect-specific + canonicalization noise (implicit casts, identifier case) that the + ORM's compiled text never has. Only called when a backend actually + reflects expression-based indexes back with real expression text to + normalize; a backend that can't (e.g. SQLite never reflects them at + all) never reaches this call. + """ + raise FeatureNotSupportedError("Functional-index expression normalization", self) + # ── Row counts ─────────────────────────────────────────────────────────── def approximate_row_counts( diff --git a/omop_alchemy/backends/postgres.py b/omop_alchemy/backends/postgres.py index 4b6b17d..970ef77 100644 --- a/omop_alchemy/backends/postgres.py +++ b/omop_alchemy/backends/postgres.py @@ -1,6 +1,7 @@ from __future__ import annotations import os +import re import shutil import sqlalchemy as sa @@ -11,6 +12,9 @@ from .base import Backend, FullTextTargetConfig +_STRING_LITERAL = re.compile(r"'(?:[^']|'')*'") +_TEXTLIKE_CAST = re.compile(r"::(?:text|varchar|character varying|bpchar|char)\b", re.IGNORECASE) + class PostgresBackend(Backend): @@ -136,6 +140,30 @@ def get_clustered_index_name( ).scalar_one_or_none() return str(result) if result is not None else None + # ── Schema reconciliation ──────────────────────────────────────────────── + + def normalize_index_expression(self, sql_text: str) -> str: + """Strip whitespace and casts to a text-ish type outside string + literals, and fold case the same way. + + Postgres's own catalog inserts these casts around string functions + as cosmetic noise when reflecting an index back, e.g. + ``lower(concept_name::text)``. A cast to any other type + (``::numeric``, ``::integer``, ...) is left intact, since that + changes the expression's actual computation. Literal contents are + never touched: identifiers, keywords, and casts are + case/whitespace-insensitive in Postgres, but a literal value isn't. + """ + parts = [] + last_end = 0 + for match in _STRING_LITERAL.finditer(sql_text): + before = sql_text[last_end : match.start()] + parts.append(_TEXTLIKE_CAST.sub("", before).replace(" ", "").lower()) + parts.append(match.group(0)) + last_end = match.end() + parts.append(_TEXTLIKE_CAST.sub("", sql_text[last_end:]).replace(" ", "").lower()) + return "".join(parts) + # ── Row counts ─────────────────────────────────────────────────────────── def approximate_row_counts( diff --git a/omop_alchemy/backends/resolve.py b/omop_alchemy/backends/resolve.py index bc84e50..afea8c8 100644 --- a/omop_alchemy/backends/resolve.py +++ b/omop_alchemy/backends/resolve.py @@ -12,7 +12,7 @@ Dialect.SQLITE: SQLiteBackend(), } -def resolve_backend(engine: sa.Engine) -> Backend: +def resolve_backend(engine: sa.Engine | sa.Connection) -> Backend: dialect = engine.dialect.name try: supported_dialect = Dialect(dialect) diff --git a/omop_alchemy/maintenance/cli_schema_reconcile.py b/omop_alchemy/maintenance/cli_schema_reconcile.py index 1069ca8..1d3ffaa 100644 --- a/omop_alchemy/maintenance/cli_schema_reconcile.py +++ b/omop_alchemy/maintenance/cli_schema_reconcile.py @@ -8,7 +8,7 @@ from oa_configurator import ResolvedDatabase, Role, find_table_in_other_schemas, supports_schemas from sqlalchemy.engine.interfaces import ReflectedForeignKeyConstraint, ReflectedIndex -from ..backends import backend_supports, resolve_backend +from ..backends import Backend, backend_supports, resolve_backend from ._cli_utils import Severity, Status from .cli_indexes import _cluster_column_names, _cluster_target_name, _find_equivalent_index from .tables import ( @@ -79,9 +79,8 @@ def _role_from_schema_tag(schema_tag: str | None) -> Role: def _effective_schema( resolved: ResolvedDatabase | None, role: Role, db_schema: str | None ) -> str | None: - """resolved.schema_for_role(role) when resolved is given, else db_schema - applied the same regardless of role. The fallback for a caller with no - resolved object to hand (e.g. a test built directly against a bare engine). + """resolved.schema_for_role(role) when given, else db_schema regardless + of role, the fallback for a caller with no resolved object to hand. """ return resolved.schema_for_role(role) if resolved is not None else db_schema @@ -91,21 +90,13 @@ def _schema_qualified_tables( ) -> dict[int, sa.Table]: """Schema-qualified copy of every table in Base.metadata, keyed by id() of the original. - Notes - ----- Each table is qualified to its own role's schema via resolved, not one - blanket value: a vocab-role table can live in a different physical schema - than a clinical one. - - Copied together into one MetaData() in a single pass: to_metadata() never - brings a referenced table along on its own, and resolving an FK's target - needs that table's copy already present in the same metadata. The whole - Base.metadata is copied, not just the selected/diffed subset, since a - selected table can reference one excluded from the diff itself (e.g. a - vocabulary FK target when vocabulary_included=False). - - Returns the tables unchanged (keyed by their own id) when both resolved - and db_schema are None. + blanket value, since a vocab-role table can live in a different physical + schema than a clinical one. Copied together into one MetaData() (not per + table): to_metadata() never brings a referenced table along on its own, + and an FK's target needs its copy already present in the same metadata. + Returns the tables unchanged, keyed by their own id, when resolved and + db_schema are both None. """ from orm_loader.helpers import Base @@ -185,6 +176,39 @@ def _actual_indexes( } +def _expected_index_signature(index: sa.Index, backend: Backend) -> tuple[str, ...]: + """Per-position signature for index: a plain column's name, or the + normalized compiled SQL text of an expression (e.g. ``func.lower(...)``), + matching how the database reflects a functional index back. Expression + normalization is dialect-specific (e.g. Postgres's own catalog inserts + casts as reflection noise), so it's delegated to backend. + """ + signature = [] + for expr in index.expressions: + if isinstance(expr, sa.Column): + signature.append(expr.name) + elif isinstance(expr, str): + signature.append(backend.normalize_index_expression(expr)) + else: + compiled = str(expr.compile(compile_kwargs={"literal_binds": True})) + signature.append(backend.normalize_index_expression(compiled)) + return tuple(signature) + + +def _actual_index_signature(actual_index: ReflectedIndex, backend: Backend) -> tuple[str, ...]: + """Per-position signature for a reflected index, matching + :func:`_expected_index_signature`'s shape. + """ + column_names = actual_index.get("column_names") or [] + if "expressions" not in actual_index: + return tuple(name for name in column_names if name is not None) + expressions = iter(actual_index.get("expressions") or []) + return tuple( + name if name is not None else backend.normalize_index_expression(next(expressions)) + for name in column_names + ) + + def reconcile_schema( engine: sa.Engine, *, @@ -192,11 +216,27 @@ def reconcile_schema( db_schema: str | None = None, vocabulary_included: bool = False, ) -> SchemaReconciliationReport: - """Compare ORM metadata against the live database schema. Reports missing columns, indexes, FKs, and cluster state. + """Compare ORM metadata against the live database schema. - resolved, when given, qualifies each table to its own role's schema - (schema_name/vocab_schema/results_schema) rather than applying db_schema - to every table regardless of role. + Parameters + ---------- + engine : sqlalchemy.Engine + Engine to inspect. Its dialect selects the backend used for + cluster-state checks. + resolved : ResolvedDatabase, optional + When given, qualifies each table to its own role's schema + (schema_name/vocab_schema/results_schema) instead of applying + db_schema to every table regardless of role. + db_schema : str, optional + Blanket schema applied to every table when resolved is not given. + vocabulary_included : bool, optional + Whether vocabulary tables are included in the diff. + + Returns + ------- + SchemaReconciliationReport + Per-table status plus every column, index, FK, and cluster issue + found. """ excluded_categories: tuple[TableCategory, ...] = ( () if vocabulary_included else (TableCategory.VOCABULARY,) @@ -451,9 +491,9 @@ def reconcile_schema( continue actual_index = actual_idxs[index_name] - expected_columns_for_index = tuple(column.name for column in index.columns) - actual_columns_for_index = tuple(c for c in (actual_index.get("column_names") or []) if c is not None) - if expected_columns_for_index != actual_columns_for_index: + expected_signature = _expected_index_signature(index, _backend) + actual_signature = _actual_index_signature(actual_index, _backend) + if expected_signature != actual_signature: table_issues.append( ReconciliationIssue( table_name=maintenance_table.table_name, @@ -461,8 +501,8 @@ def reconcile_schema( component="index", object_name=index_name, status=Status.MISMATCH, - expected=", ".join(expected_columns_for_index), - actual=", ".join(actual_columns_for_index) if actual_columns_for_index else None, + expected=", ".join(expected_signature), + actual=", ".join(actual_signature) if actual_signature else None, detail="Index columns differ from ORM metadata.", ) ) diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index 11a0273..10e1b74 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -216,18 +216,7 @@ def test_reconcile_schema_with_resolved_qualifies_each_table_to_its_own_role_sch report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) - # index issues are excluded here: ix_concept_concept_name_lower is a - # functional index (lower(concept_name)), and reconcile_schema's - # index diff can't read its expression column back from the - # inspector. Confirmed pre-existing and schema-independent (it - # reproduces against a single non-default schema too), so it is out - # of scope for this test, which only covers cross-schema behaviour. - # cluster issues are no longer excluded: reconcile_schema's cluster - # check used to call get_clustered_index_name() without a role, - # always inspecting the primary schema, so a vocab/results table's - # real cluster state was invisible whenever its schema differed from - # primary. Fixed by passing role=table_role through. - checked_components = {"table", "column", "primary_key", "foreign_key", "cluster"} + checked_components = {"table", "column", "primary_key", "foreign_key", "cluster", "index"} for table_name in ("person", "concept", "observation_period"): issues = [ issue @@ -237,6 +226,45 @@ def test_reconcile_schema_with_resolved_qualifies_each_table_to_its_own_role_sch assert issues == [], (table_name, issues) +@pytest.mark.postgresql +@pytest.mark.db_dialect +def test_reconcile_schema_catches_genuine_drift_in_a_functional_index(pg_db, pg_engine): + """concept's ix_concept_concept_name_lower (a functional index, + lower(concept_name)) reports no drift when unchanged, and a real + MISMATCH when its live expression is deliberately altered. Proves the + normalization process compares signatures rather than just silencing the check. + """ + with isolated_test_schema(pg_engine, prefix="reconcile_functional_index") as schema: + resolved = dataclasses.replace( + pg_db.resolved, schema_name=schema, vocab_schema=schema, results_schema=schema + ) + engine = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema, "vocab": schema, "results": schema} + ) + create_missing_tables(engine, resolved=resolved) + + def _index_issues(report): + return [ + issue + for issue in report.issues + if issue.table_name == "concept" and issue.object_name == "ix_concept_concept_name_lower" + ] + + report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) + assert _index_issues(report) == [] + + with engine.begin() as connection: + connection.exec_driver_sql(f'DROP INDEX {qualified(connection, "ix_concept_concept_name_lower")}') + connection.exec_driver_sql( + f'CREATE INDEX ix_concept_concept_name_lower ON {qualified(connection, "concept")} (upper(concept_name))' + ) + + report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) + issues = _index_issues(report) + assert len(issues) == 1 + assert issues[0].status == "mismatch" + + def test_is_blocking_issue_excludes_renamed_only(): from omop_alchemy.maintenance.cli_schema_reconcile import ReconciliationIssue from omop_alchemy.maintenance._cli_utils import Status @@ -349,3 +377,25 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_tar assert cluster_issues[0].actual == "idx_person_id" assert unexpected_index_issues == [] assert person_result.status == "matched" + + +@pytest.mark.parametrize( + ("a", "b", "should_match"), + [ + pytest.param("lower(x)", "lower(x::text)", True, id="cosmetic-textlike-cast-ignored"), + pytest.param("LOWER(x)", "lower(x::text)", True, id="function-name-case-ignored"), + pytest.param("lower( x )", "lower(x::text)", True, id="incidental-whitespace-ignored"), + pytest.param("sum(x::numeric)", "sum(x::integer)", False, id="meaningful-cast-still-caught"), + pytest.param( + "coalesce(x, 'Unknown')", "coalesce(x, 'unknown')", False, id="literal-case-still-caught" + ), + pytest.param( + "concat_ws(' - ', a, b)", "concat_ws('-', a, b)", False, id="literal-whitespace-still-caught" + ), + ], +) +def test_normalize_index_expression_ignores_cosmetic_noise_but_not_real_drift(a, b, should_match): + from omop_alchemy.backends.postgres import PostgresBackend + + backend = PostgresBackend() + assert (backend.normalize_index_expression(a) == backend.normalize_index_expression(b)) is should_match From 4e4fb8627dafae164b8a7fb4e2de5f9fa1e316c8 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 04:31:19 +0000 Subject: [PATCH 19/32] Resolve fulltext tables base on the role of the table instead of hardcoding vocab --- omop_alchemy/maintenance/cli_fulltext.py | 27 ++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/omop_alchemy/maintenance/cli_fulltext.py b/omop_alchemy/maintenance/cli_fulltext.py index 6c41113..2f20677 100644 --- a/omop_alchemy/maintenance/cli_fulltext.py +++ b/omop_alchemy/maintenance/cli_fulltext.py @@ -4,14 +4,19 @@ from dataclasses import dataclass from enum import StrEnum +from typing import cast import typer +import sqlalchemy as sa from sqlalchemy.engine import Engine - from oa_configurator import Role +from orm_loader.helpers import role_of_table + from ..backends import backend_support_note as _backend_support_note from ..backends import resolve_backend, require_backend_support from ..backends.base import FullTextError +from ..cdm.model.vocabulary.concept import Concept +from ..cdm.model.vocabulary.concept_synonym import Concept_Synonym from ._cli_utils import Status, dry_label, dry_status, omop_command from .ui import ( console, @@ -19,6 +24,20 @@ render_fulltext_summary, ) +_FULLTEXT_TARGET_TABLES: dict[str, sa.Table] = { + "concept": cast(sa.Table, Concept.__table__), + "concept_synonym": cast(sa.Table, Concept_Synonym.__table__), +} + + +def _role_for_target(table_name: str) -> Role: + """The Role a fulltext target table's own declared schema tag names. + Resolves via role_of_table() rather than hardcoding Role.VOCAB, + so a future non-vocab fulltext target resolves correctly. + """ + return role_of_table(_FULLTEXT_TARGET_TABLES[table_name]) + + app = typer.Typer( help=f"Manage full-text search for OMOP vocabulary tables. {_backend_support_note('install_fulltext_on_table')}", rich_markup_mode="rich", @@ -72,7 +91,7 @@ def install_fulltext_columns( index_name=cfg.index_name, create_indexes=create_indexes, fastupdate=fastupdate, - role=Role.VOCAB, + role=_role_for_target(cfg.table_name), ) backend.register_fulltext_metadata() except FullTextError: @@ -123,7 +142,7 @@ def populate_fulltext_columns( vector_column_name=cfg.vector_column_name, source_column_name=cfg.source_column_name, regconfig=regconfig, - role=Role.VOCAB, + role=_role_for_target(cfg.table_name), ) backend.register_fulltext_metadata() except FullTextError: @@ -170,7 +189,7 @@ def drop_fulltext_columns( vector_column_name=cfg.vector_column_name, index_name=cfg.index_name, drop_indexes=drop_indexes, - role=Role.VOCAB, + role=_role_for_target(cfg.table_name), ) backend.unregister_fulltext_metadata() except FullTextError: From 13702342ee328b989da2c8ac2e5e8cff767176ef Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 04:36:18 +0000 Subject: [PATCH 20/32] Re-use common fixture --- tests/conftest.py | 31 ++++++++++++++++++- tests/test_schema_reconcile.py | 20 +++--------- ...test_vocab_results_role_wiring_postgres.py | 16 +++------- 3 files changed, 39 insertions(+), 28 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 879c438..23fc67f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import typer.rich_utils as _typer_rich_utils from orm_loader.helpers import bootstrap from oa_configurator.testing import isolated_test_database, isolated_test_schema -from oa_configurator import SCHEMA_TRANSLATE_MAP_KEY, Role +from oa_configurator import SCHEMA_TRANSLATE_MAP_KEY, Role, ResolvedCDMDatabase, ResolvedConnection import sqlalchemy.orm as so from sqlalchemy.orm import Session, sessionmaker @@ -36,6 +36,35 @@ _typer_rich_utils.FORCE_TERMINAL = None +def resolved_cdm_database_from_engine( + engine: sa.Engine, + *, + name: str, + schema_name: str | None = None, + vocab_schema: str | None = None, + results_schema: str | None = None, +) -> ResolvedCDMDatabase: + """Build a ResolvedCDMDatabase from an already-live engine's own + URL, for a test that needs a resolved object pointing at one real + engine with caller-chosen schema names. + """ + url = engine.url + connection = ResolvedConnection( + name=name, + url=url.render_as_string(hide_password=False), + safe_url=url.render_as_string(hide_password=True), + _engine_url=url, + ) + return ResolvedCDMDatabase( + name=name, + connection=connection, + schema_name=schema_name, + vocab_connection=connection, + vocab_schema=vocab_schema, + results_schema=results_schema, + ) + + @pytest.fixture def fresh_engine() -> Iterator[sa.Engine]: """Fresh, empty, function-scoped SQLite engine. diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index 10e1b74..5216664 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -4,7 +4,7 @@ import pytest import sqlalchemy as sa -from oa_configurator import ResolvedCDMDatabase, ResolvedConnection, Role +from oa_configurator import ResolvedCDMDatabase, Role from oa_configurator import qualified, schema_of, Dialect from oa_configurator.testing import DIALECT_PARAMS, isolated_test_schema from omop_alchemy.backends.sqlite import SQLiteBackend @@ -13,6 +13,8 @@ from omop_alchemy.maintenance.cli_schema import create_missing_tables from omop_alchemy.maintenance.cli_schema_reconcile import is_blocking_issue, reconcile_schema +from tests.conftest import resolved_cdm_database_from_engine + PERSON_GENDER_INDEX = omop_index_name("person", "gender_concept_id") EPISODE_PERSON_INDEX = omop_index_name("episode", "person_id") @@ -28,21 +30,7 @@ def _sqlite_resolved(engine: sa.Engine) -> ResolvedCDMDatabase: connection identity to resolve, so building this directly from the engine's own URL is the whole story, unlike Postgres. """ - url = engine.url - connection = ResolvedConnection( - name="fresh_engine_test", - url=url.render_as_string(hide_password=False), - safe_url=url.render_as_string(hide_password=True), - _engine_url=url, - ) - return ResolvedCDMDatabase( - name="fresh_engine_test", - connection=connection, - schema_name=None, - vocab_connection=connection, - vocab_schema=None, - results_schema=None, - ) + return resolved_cdm_database_from_engine(engine, name="fresh_engine_test") @pytest.fixture(params=DIALECT_PARAMS) diff --git a/tests/test_vocab_results_role_wiring_postgres.py b/tests/test_vocab_results_role_wiring_postgres.py index 3eb15bc..7e0e608 100644 --- a/tests/test_vocab_results_role_wiring_postgres.py +++ b/tests/test_vocab_results_role_wiring_postgres.py @@ -24,9 +24,11 @@ import sqlalchemy as sa import sqlalchemy.orm as so -from oa_configurator import ResolvedCDMDatabase, ResolvedConnection, Role +from oa_configurator import Role from oa_configurator.testing import isolated_test_schema +from tests.conftest import resolved_cdm_database_from_engine + from omop_alchemy.cdm.model.clinical import Observation, Person from omop_alchemy.cdm.model.derived import Cohort from omop_alchemy.cdm.model.vocabulary import Concept, Concept_Class, Domain, Vocabulary @@ -208,18 +210,10 @@ def _drop_schemas() -> None: cleanup_after_test(_drop_schemas) - url = pg_engine.url - connection = ResolvedConnection( - name="fresh_schema_test", - url=url.render_as_string(hide_password=False), - safe_url=url.render_as_string(hide_password=True), - _engine_url=url, - ) - resolved = ResolvedCDMDatabase( + resolved = resolved_cdm_database_from_engine( + pg_engine, name="fresh_schema_test", - connection=connection, schema_name=clinical_schema, - vocab_connection=connection, vocab_schema=vocab_schema, results_schema=results_schema, ) From 65900e4ad3329355f14f72a8e88c08ba03f78168 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 23:37:48 +0000 Subject: [PATCH 21/32] Correctly classify partial indexes --- omop_alchemy/maintenance/cli_indexes.py | 39 ++++++++++++++++++++--- tests/test_indexes.py | 42 +++++++++++++++++++------ 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index 23fc32a..963c8ef 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -81,13 +81,41 @@ def _is_plain_index(reflected: Mapping[str, Any]) -> bool: if reflected.get("duplicates_constraint"): return False dialect_options = reflected.get("dialect_options") or {} - if dialect_options.get("postgresql_where"): + if _dialect_option(dialect_options, "where") is not None: return False - if dialect_options.get("postgresql_using"): + if _dialect_option(dialect_options, "using") is not None: return False return True +def _dialect_option(dialect_options: Mapping[str, Any], suffix: str) -> Any | None: + """Find a reflected index's dialect-specific option, regardless of dialect. + + SQLAlchemy always prefixes a reflected index's dialect-specific options + with the dialect name, e.g. ``"postgresql_where"`` or ``"sqlite_where"``. + Matching by suffix instead of a hardcoded dialect name means this works + for any dialect, current or future, with no per-dialect registration + needed. + + Parameters + ---------- + dialect_options : Mapping[str, Any] + A reflected index's ``dialect_options`` mapping. + suffix : str + The option name to look for, without its dialect prefix (e.g. + ``"where"``, ``"using"``). + + Returns + ------- + Any | None + The matching option's value, or None if no dialect set it. + """ + for key, value in dialect_options.items(): + if key.endswith(f"_{suffix}") and value: + return value + return None + + def _find_equivalent_index( existing_indexes: Sequence[Mapping[str, Any]], column_names: tuple[str, ...], @@ -185,10 +213,11 @@ def _describe_shape_conflict(reflected: Mapping[str, Any]) -> str: reasons: list[str] = [] if reflected.get("duplicates_constraint"): reasons.append("backs a UNIQUE/PRIMARY KEY constraint") - if dialect_options.get("postgresql_where"): + if _dialect_option(dialect_options, "where") is not None: reasons.append("has a partial WHERE predicate") - if dialect_options.get("postgresql_using"): - reasons.append(f"uses non-btree access method '{dialect_options['postgresql_using']}'") + using = _dialect_option(dialect_options, "using") + if using is not None: + reasons.append(f"uses non-btree access method '{using}'") if not reasons: reasons.append("has an unsupported definition") return ", ".join(reasons) diff --git a/tests/test_indexes.py b/tests/test_indexes.py index db9ec17..440350d 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -532,6 +532,20 @@ def test_is_plain_index_false_for_non_btree_index(): assert _is_plain_index(_NON_BTREE) is False +def test_is_plain_index_false_for_sqlite_partial_index(): + """A SQLite partial index, reflected with a sqlite_where dialect + option, must not be classified as plain. Treating it as plain would + make it look safe to drop and recreate from a bare column list, which + would silently lose its WHERE predicate.""" + sqlite_partial = { + "name": "idx_partial_sqlite", + "column_names": ["gender_concept_id"], + "unique": False, + "dialect_options": {"sqlite_where": "gender_concept_id IS NOT NULL"}, + } + assert _is_plain_index(sqlite_partial) is False + + def test_find_equivalent_index_is_order_sensitive(): existing = [{"name": "idx_ab", "column_names": ["b", "a"], "unique": False}] assert _find_equivalent_index(existing, ("a", "b"), False) is None @@ -562,6 +576,20 @@ def test_describe_shape_conflict_mentions_reason(): assert "non-btree access method 'gin'" in _describe_shape_conflict(_NON_BTREE) +def test_describe_shape_conflict_mentions_reason_for_sqlite_dialect_options(): + """The conflict reason for a SQLite index must name the actual cause + (a WHERE predicate or a non-btree access method), read from its + sqlite_where/sqlite_using dialect options, not just Postgres's.""" + sqlite_partial = { + "dialect_options": {"sqlite_where": "gender_concept_id IS NOT NULL"}, + } + sqlite_non_btree = { + "dialect_options": {"sqlite_using": "gin"}, + } + assert "partial WHERE predicate" in _describe_shape_conflict(sqlite_partial) + assert "non-btree access method 'gin'" in _describe_shape_conflict(sqlite_non_btree) + + # ── Reserved schema guard ──────────────────────────────────────────────────────── # The check itself lives in oa_configurator (resolved via CDMDatabaseConfig.resolve(), # see oa-configurator's own test_resolver.py::TestReservedSchemaCollision). This @@ -923,8 +951,7 @@ def test_record_captured_index_scopes_by_db_schema(indexed_engine): assert captured_b is True # Capturing again for the *same* schema, with a different foreign name, must - # still be rejected (this is the case test_manage_indexes_disable_second_run_ - # without_enable_degrades_to_warning covers end-to-end). + # still be rejected. with engine.begin() as connection: captured_a_again = _record_captured_index( connection, @@ -966,14 +993,9 @@ def test_resolve_physical_cluster_name_ignores_uniqueness_for_pk_based_target(): def test_vocabulary_domain_concept_class_relationship_cluster_on_primary_key(): - """vocabulary/domain/concept_class/relationship previously declared a - redundant secondary index as their cluster target (same column as their own - primary key), inconsistently with person/location/care_site/provider/concept - which cluster directly on the primary key's own index. Normalized onto the - latter: Alchemy never creates that redundant index itself, and index - reconciliation is what recognizes a database that has the OHDSI-standard - duplicate (see _resolve_physical_cluster_name_falls_back_to_equivalent) - as an equivalent cluster target.""" + """vocabulary/domain/concept_class/relationship must cluster directly on + their own primary key's index, the same as person/location/care_site/ + provider/concept do, not on a separate, redundant same-column index.""" tables = {table.table_name: table for table in collect_maintenance_tables()} for table_name, pk_column in ( ("vocabulary", "vocabulary_id"), From c9d7fcc087757998d58380156a253a6767c92785 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 23:38:30 +0000 Subject: [PATCH 22/32] PRoperly dispatch to dialect resolving --- omop_alchemy/maintenance/cli_vocab.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/omop_alchemy/maintenance/cli_vocab.py b/omop_alchemy/maintenance/cli_vocab.py index f524646..5ba264a 100644 --- a/omop_alchemy/maintenance/cli_vocab.py +++ b/omop_alchemy/maintenance/cli_vocab.py @@ -38,6 +38,7 @@ Vocabulary, ) +from ..backends import backend_supports, resolve_backend as resolve_omop_backend from ._cli_utils import Status, omop_command from .cli_foreign_keys import manage_foreign_key_triggers from .cli_indexes import manage_indexes @@ -564,7 +565,7 @@ def load_vocab_source( table_count=table_count, ) - if not dry_run and engine.dialect.name == Dialect.POSTGRESQL: + if not dry_run and backend_supports(resolve_omop_backend(engine), "find_sequence_name"): sequence_results = reset_model_sequences( engine, vocabulary_included=True, From a211efb4726b49c2fed81a350769308dedcd2c57 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Tue, 15 Sep 2026 23:40:19 +0000 Subject: [PATCH 23/32] Utilise dataclasses.replace to properly setup tests --- tests/conftest.py | 8 +++++--- tests/test_load_vocab_source.py | 9 +++++++++ tests/test_vocab_results_role_wiring_postgres.py | 12 +++++++----- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 23fc67f..3323bf6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -44,9 +44,11 @@ def resolved_cdm_database_from_engine( vocab_schema: str | None = None, results_schema: str | None = None, ) -> ResolvedCDMDatabase: - """Build a ResolvedCDMDatabase from an already-live engine's own - URL, for a test that needs a resolved object pointing at one real - engine with caller-chosen schema names. + """Build a ResolvedCDMDatabase from an already-live engine's own URL. + + Only for a case with no resolved object to build off at all (e.g. a + bare SQLite engine). When one already exists, prefer + ``dataclasses.replace(existing.resolved, ...)`` instead. """ url = engine.url connection = ResolvedConnection( diff --git a/tests/test_load_vocab_source.py b/tests/test_load_vocab_source.py index c4cdb11..d92a8be 100644 --- a/tests/test_load_vocab_source.py +++ b/tests/test_load_vocab_source.py @@ -793,3 +793,12 @@ def test_render_vocab_index_warnings_lists_messages_when_present(): summary_text = summary_buffer.getvalue() assert "Index warnings" in summary_text assert "1" in summary_text + + +def test_sequence_reset_gate_matches_find_sequence_name_capability(fresh_engine, pg_engine): + """The sequence-reset gate must match find_sequence_name support: + False for SQLite, True for Postgres.""" + from omop_alchemy.backends import backend_supports, resolve_backend + + assert backend_supports(resolve_backend(fresh_engine), "find_sequence_name") is False + assert backend_supports(resolve_backend(pg_engine), "find_sequence_name") is True diff --git a/tests/test_vocab_results_role_wiring_postgres.py b/tests/test_vocab_results_role_wiring_postgres.py index 7e0e608..2ce0b35 100644 --- a/tests/test_vocab_results_role_wiring_postgres.py +++ b/tests/test_vocab_results_role_wiring_postgres.py @@ -15,6 +15,7 @@ from __future__ import annotations +import dataclasses import uuid from contextlib import ExitStack from datetime import date @@ -27,8 +28,6 @@ from oa_configurator import Role from oa_configurator.testing import isolated_test_schema -from tests.conftest import resolved_cdm_database_from_engine - from omop_alchemy.cdm.model.clinical import Observation, Person from omop_alchemy.cdm.model.derived import Cohort from omop_alchemy.cdm.model.vocabulary import Concept, Concept_Class, Domain, Vocabulary @@ -189,7 +188,7 @@ def test_clinical_to_vocab_join_compiles_and_executes_in_one_query( def test_create_missing_tables_creates_vocab_and_results_schemas_on_a_fresh_database( - pg_engine: sa.Engine, cleanup_after_test + pg_db, pg_engine: sa.Engine, cleanup_after_test ) -> None: """create_missing_tables() used to call ensure_schema() only for the primary schema, so a genuinely fresh database (where vocab/results @@ -210,12 +209,15 @@ def _drop_schemas() -> None: cleanup_after_test(_drop_schemas) - resolved = resolved_cdm_database_from_engine( - pg_engine, + patched_connection = dataclasses.replace(pg_db.resolved.connection, test_only=False) + resolved = dataclasses.replace( + pg_db.resolved, name="fresh_schema_test", schema_name=clinical_schema, vocab_schema=vocab_schema, results_schema=results_schema, + connection=patched_connection, + vocab_connection=patched_connection, ) engine = pg_engine.execution_options( schema_translate_map={ From 6d6ba784d466152d896dd74df0f8a8973626d6a0 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Wed, 16 Sep 2026 03:03:13 +0000 Subject: [PATCH 24/32] Alleviate last two instance of incomplete dialect-split --- omop_alchemy/maintenance/cli_schema_doctor.py | 18 ++++++++++++++---- omop_alchemy/maintenance/cli_vocab.py | 4 ++-- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/omop_alchemy/maintenance/cli_schema_doctor.py b/omop_alchemy/maintenance/cli_schema_doctor.py index 7ea634c..eb60395 100644 --- a/omop_alchemy/maintenance/cli_schema_doctor.py +++ b/omop_alchemy/maintenance/cli_schema_doctor.py @@ -7,6 +7,7 @@ import sqlalchemy as sa from oa_configurator import Dialect, ResolvedDatabase +from ..backends import backend_supports, resolve_backend from ._cli_utils import Status from .cli_foreign_keys import ( ForeignKeyStatusResult, @@ -271,7 +272,8 @@ def collect_doctor_report( ) ) - if info.backend == Dialect.POSTGRESQL: + backend = resolve_backend(engine) + if backend_supports(backend, "get_fk_trigger_counts"): foreign_key_status = tuple( collect_foreign_key_trigger_status( engine, @@ -295,7 +297,7 @@ def collect_doctor_report( ) ) - if deep: + if deep and backend_supports(backend, "count_fk_violations"): foreign_key_validation = validate_foreign_key_constraints( engine, vocabulary_included=vocabulary_included, @@ -319,6 +321,14 @@ def collect_doctor_report( ), ) ) + elif deep: + checks.append( + DoctorCheck( + name="foreign key validation", + status=Status.SKIPPED, + detail="Foreign key validation isn't supported on this backend.", + ) + ) else: checks.append( DoctorCheck( @@ -332,14 +342,14 @@ def collect_doctor_report( DoctorCheck( name="foreign keys", status=Status.SKIPPED, - detail="Foreign key trigger inspection is only available on PostgreSQL.", + detail="Foreign key trigger inspection isn't supported on this backend.", ) ) checks.append( DoctorCheck( name="foreign key validation", status=Status.SKIPPED, - detail="Foreign key validation is only available on PostgreSQL.", + detail="Foreign key validation isn't supported on this backend.", ) ) else: diff --git a/omop_alchemy/maintenance/cli_vocab.py b/omop_alchemy/maintenance/cli_vocab.py index 5ba264a..b1fcf92 100644 --- a/omop_alchemy/maintenance/cli_vocab.py +++ b/omop_alchemy/maintenance/cli_vocab.py @@ -12,7 +12,7 @@ import sqlalchemy.orm as so from sqlalchemy.exc import OperationalError import typer -from oa_configurator import Dialect, ensure_schema +from oa_configurator import ensure_schema from orm_loader.backends import STAGING_SCHEMA, resolve_backend from orm_loader.helpers import Base from orm_loader.tables.typing import CSVTableProtocol @@ -365,7 +365,7 @@ def load_vocab_source( ) _use_bulk_mode = ( - bulk_mode and not dry_run and engine.dialect.name == Dialect.POSTGRESQL + bulk_mode and not dry_run and backend_supports(resolve_omop_backend(engine), "toggle_fk_triggers") ) if _use_bulk_mode: _emit( From 5f7a2d983a3f7d3b2b20d84e028228387db53bb4 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Mon, 21 Sep 2026 01:04:11 +0000 Subject: [PATCH 25/32] Correctly guard vocab, pull role_of_table from oa-configurator --- omop_alchemy/maintenance/cli_fulltext.py | 3 +-- .../maintenance/cli_schema_reconcile.py | 25 ++++++++----------- omop_alchemy/maintenance/cli_schema_tables.py | 1 + omop_alchemy/maintenance/tables.py | 3 +-- 4 files changed, 14 insertions(+), 18 deletions(-) diff --git a/omop_alchemy/maintenance/cli_fulltext.py b/omop_alchemy/maintenance/cli_fulltext.py index 2f20677..c8f3940 100644 --- a/omop_alchemy/maintenance/cli_fulltext.py +++ b/omop_alchemy/maintenance/cli_fulltext.py @@ -9,8 +9,7 @@ import typer import sqlalchemy as sa from sqlalchemy.engine import Engine -from oa_configurator import Role -from orm_loader.helpers import role_of_table +from oa_configurator import Role, role_of_table from ..backends import backend_support_note as _backend_support_note from ..backends import resolve_backend, require_backend_support diff --git a/omop_alchemy/maintenance/cli_schema_reconcile.py b/omop_alchemy/maintenance/cli_schema_reconcile.py index 1d3ffaa..5274a30 100644 --- a/omop_alchemy/maintenance/cli_schema_reconcile.py +++ b/omop_alchemy/maintenance/cli_schema_reconcile.py @@ -5,7 +5,13 @@ from dataclasses import dataclass import sqlalchemy as sa -from oa_configurator import ResolvedDatabase, Role, find_table_in_other_schemas, supports_schemas +from oa_configurator import ( + ResolvedDatabase, + Role, + find_table_in_other_schemas, + role_of_table, + supports_schemas +) from sqlalchemy.engine.interfaces import ReflectedForeignKeyConstraint, ReflectedIndex from ..backends import Backend, backend_supports, resolve_backend @@ -67,15 +73,6 @@ class SchemaReconciliationReport: issues: tuple[ReconciliationIssue, ...] -def _role_from_schema_tag(schema_tag: str | None) -> Role: - """Map a table's declared .schema tag (None/"vocab"/"results") to its Role.""" - if schema_tag == Role.VOCAB.value: - return Role.VOCAB - if schema_tag == Role.RESULTS.value: - return Role.RESULTS - return Role.PRIMARY - - def _effective_schema( resolved: ResolvedDatabase | None, role: Role, db_schema: str | None ) -> str | None: @@ -106,7 +103,7 @@ def _schema_qualified_tables( def _referred_schema(_table: sa.Table, _to_schema, _constraint, referred_schema: str | None): # None means "unchanged" to to_metadata(); BLANK_SCHEMA is what actually clears a schema tag. - target = _effective_schema(resolved, _role_from_schema_tag(referred_schema), db_schema) + target = _effective_schema(resolved, Role(referred_schema), db_schema) return target if target is not None else sa.BLANK_SCHEMA return { @@ -114,7 +111,7 @@ def _referred_schema(_table: sa.Table, _to_schema, _constraint, referred_schema: metadata, # SQLAlchemy's own stub omits None from schema's declared type, # despite accepting and correctly handling it at runtime - schema=_effective_schema(resolved, _role_from_schema_tag(table.schema), db_schema), # ty: ignore[invalid-argument-type] + schema=_effective_schema(resolved, role_of_table(table), db_schema), # ty: ignore[invalid-argument-type] referred_schema_fn=_referred_schema, ) for table in Base.metadata.tables.values() @@ -252,7 +249,7 @@ def reconcile_schema( with engine.connect() as connection: for maintenance_table in selected_tables: table_issues: list[ReconciliationIssue] = [] - table_role = _role_from_schema_tag(maintenance_table.table.schema) + table_role = role_of_table(maintenance_table.table) table_schema = _effective_schema(resolved, table_role, db_schema) exists = inspector.has_table(maintenance_table.table_name, schema=table_schema) if not exists: @@ -413,7 +410,7 @@ def reconcile_schema( if ( not _cross_schema_fk_supported and raw_constraint is not None - and _role_from_schema_tag(raw_constraint.referred_table.schema) != table_role + and role_of_table(raw_constraint.referred_table) != table_role ): # SQLite can never create an inline FK crossing a schema boundary. continue diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index 92e4bc2..be127b3 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -128,6 +128,7 @@ def create_missing_tables( engine.begin() as connection, guard_schema_provenance(connection, resolved, role=Role.PRIMARY), guard_schema_provenance(connection, resolved, role=Role.RESULTS), + guard_schema_provenance(connection, resolved, role=Role.VOCAB), ): Base.metadata.create_all( bind=connection, tables=all_tables, checkfirst=True diff --git a/omop_alchemy/maintenance/tables.py b/omop_alchemy/maintenance/tables.py index d24023f..1a0ff11 100644 --- a/omop_alchemy/maintenance/tables.py +++ b/omop_alchemy/maintenance/tables.py @@ -5,8 +5,7 @@ from typing import Iterable import sqlalchemy as sa -from oa_configurator import Role, schema_of -from orm_loader.helpers import role_of_table +from oa_configurator import Role, role_of_table, schema_of class TableCategory(StrEnum): From 5ee4b9b29457d8107e9dce75c64109496f440231 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Wed, 23 Sep 2026 00:35:08 +0000 Subject: [PATCH 26/32] Role removal following oa-configurator --- omop_alchemy/backends/base.py | 36 ++++----- omop_alchemy/backends/postgres.py | 68 ++++++++--------- omop_alchemy/backends/sqlite.py | 4 +- omop_alchemy/cdm/base/decorators.py | 14 ++++ omop_alchemy/config.py | 2 +- omop_alchemy/maintenance/cli_foreign_keys.py | 28 +++---- omop_alchemy/maintenance/cli_fulltext.py | 29 ++++++-- omop_alchemy/maintenance/cli_indexes.py | 48 +++++++----- .../maintenance/cli_schema_reconcile.py | 58 ++++++++------- .../maintenance/cli_schema_summary.py | 10 +-- omop_alchemy/maintenance/cli_schema_tables.py | 44 +++++------ omop_alchemy/maintenance/cli_tables.py | 73 +++++++++++-------- omop_alchemy/maintenance/cli_vocab.py | 4 +- omop_alchemy/maintenance/tables.py | 29 ++++---- tests/conftest.py | 8 +- tests/test_foreign_keys.py | 16 ++-- tests/test_fulltext.py | 1 + tests/test_indexes.py | 25 ++++--- tests/test_load_vocab_source.py | 4 +- tests/test_schema_reconcile.py | 18 ++--- tests/test_truncate_tables.py | 3 +- 21 files changed, 288 insertions(+), 234 deletions(-) diff --git a/omop_alchemy/backends/base.py b/omop_alchemy/backends/base.py index 937ac7d..1b31295 100644 --- a/omop_alchemy/backends/base.py +++ b/omop_alchemy/backends/base.py @@ -86,7 +86,7 @@ def toggle_fk_triggers( table_name: str, *, enable: bool, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("FK trigger management", self) @@ -95,7 +95,7 @@ def get_fk_trigger_counts( conn: sa.Connection, table_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[int, int]: """Return (disabled_count, enabled_count) for RI triggers on the table.""" raise FeatureNotSupportedError("FK trigger status inspection", self) @@ -108,8 +108,8 @@ def count_fk_violations( constrained_cols: list[str], referred_cols: list[str], *, - source_role: Role = Role.PRIMARY, - referred_role: Role = Role.PRIMARY, + source_schema_tag: str = Role.PRIMARY.value, + referred_schema_tag: str = Role.PRIMARY.value, ) -> int: raise FeatureNotSupportedError("FK constraint violation counting", self) @@ -121,7 +121,7 @@ def cluster_table( table_name: str, index_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("Table clustering", self) @@ -130,7 +130,7 @@ def get_clustered_index_name( conn: sa.Connection, table_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> str | None: raise FeatureNotSupportedError("Cluster index inspection", self) @@ -173,7 +173,7 @@ def analyze_table( table_name: str, *, vacuum: bool = False, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: ... def index_exists( @@ -181,7 +181,7 @@ def index_exists( conn: sa.Connection, index_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> bool: """Return True when the named index currently exists on the database. @@ -195,14 +195,14 @@ def drop_index_if_exists( conn: sa.Connection, index_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: """Drop an index by name without relying on SQLAlchemy's reflection-based checkfirst. Some backends (e.g. SQLite) can't reflect expression-based indexes, so Index.drop(checkfirst=True) would silently no-op on them. IF EXISTS is - evaluated by the database itself, not by reflection. role is accepted - for interface parity with the schema-aware override + evaluated by the database itself, not by reflection. schema_tag is + accepted for interface parity with the schema-aware override (PostgresBackend); this default implementation is unqualified, matching SQLite having no schema concept to route through. """ @@ -215,7 +215,7 @@ def truncate_table_batch( *, restart_identities: bool, cascade: bool, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("TRUNCATE with RESTART IDENTITY / CASCADE", self) @@ -227,7 +227,7 @@ def find_sequence_name( table_name: str, column_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> str | None: raise FeatureNotSupportedError("Owned sequence lookup", self) @@ -275,7 +275,7 @@ def install_fulltext_on_table( index_name: str, create_indexes: bool, fastupdate: bool, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("Full-text search", self) @@ -287,7 +287,7 @@ def populate_fulltext_on_table( vector_column_name: str, source_column_name: str, regconfig: str, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> int | None: raise FeatureNotSupportedError("Full-text search", self) @@ -299,7 +299,7 @@ def drop_fulltext_on_table( vector_column_name: str, index_name: str, drop_indexes: bool, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: raise FeatureNotSupportedError("Full-text search", self) @@ -311,7 +311,7 @@ def prepare_backup( output_path: str, backup_format: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[str, list[str], dict[str, str], str]: """Return (tool_path, command, env, database_name). subprocess.run stays in CLI.""" raise FeatureNotSupportedError("Database backup", self) @@ -322,7 +322,7 @@ def prepare_restore( input_path: str, backup_format: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[str, list[str], dict[str, str], str]: """Return (tool_path, command, env, database_name). subprocess.run stays in CLI.""" raise FeatureNotSupportedError("Database restore", self) diff --git a/omop_alchemy/backends/postgres.py b/omop_alchemy/backends/postgres.py index 970ef77..14c2791 100644 --- a/omop_alchemy/backends/postgres.py +++ b/omop_alchemy/backends/postgres.py @@ -34,11 +34,11 @@ def toggle_fk_triggers( table_name: str, *, enable: bool, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: action = "ENABLE" if enable else "DISABLE" conn.exec_driver_sql( - f"ALTER TABLE {qualified(conn, table_name, role=role)} {action} TRIGGER ALL" + f"ALTER TABLE {qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag))} {action} TRIGGER ALL" ) def get_fk_trigger_counts( @@ -46,7 +46,7 @@ def get_fk_trigger_counts( conn: sa.Connection, table_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[int, int]: disabled_count, enabled_count = conn.execute( sa.text( @@ -63,7 +63,7 @@ def get_fk_trigger_counts( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": schema_of(conn, role=role)}, + {"table_name": table_name, "db_schema": schema_of(conn, schema_tag=schema_tag)}, ).one() return int(disabled_count or 0), int(enabled_count or 0) @@ -75,11 +75,11 @@ def count_fk_violations( constrained_cols: list[str], referred_cols: list[str], *, - source_role: Role = Role.PRIMARY, - referred_role: Role = Role.PRIMARY, + source_schema_tag: str = Role.PRIMARY.value, + referred_schema_tag: str = Role.PRIMARY.value, ) -> int: - source = qualified(conn, source_table, role=source_role) - referred = qualified(conn, referred_table, role=referred_role) + source = qualified(conn, source_table, physical_schema=schema_of(conn, schema_tag=source_schema_tag)) + referred = qualified(conn, referred_table, physical_schema=schema_of(conn, schema_tag=referred_schema_tag)) non_null_predicate = " AND ".join( f"src.{col} IS NOT NULL" for col in constrained_cols ) @@ -110,10 +110,10 @@ def cluster_table( table_name: str, index_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: conn.exec_driver_sql( - f"CLUSTER {qualified(conn, table_name, role=role)} USING {index_name}" + f"CLUSTER {qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag))} USING {index_name}" ) def get_clustered_index_name( @@ -121,7 +121,7 @@ def get_clustered_index_name( conn: sa.Connection, table_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> str | None: result = conn.execute( sa.text( @@ -136,7 +136,7 @@ def get_clustered_index_name( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": schema_of(conn, role=role)}, + {"table_name": table_name, "db_schema": schema_of(conn, schema_tag=schema_tag)}, ).scalar_one_or_none() return str(result) if result is not None else None @@ -187,19 +187,19 @@ def analyze_table( table_name: str, *, vacuum: bool = False, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: operation = "VACUUM ANALYZE" if vacuum else "ANALYZE" - conn.exec_driver_sql(f"{operation} {qualified(conn, table_name, role=role)}") + conn.exec_driver_sql(f"{operation} {qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag))}") def index_exists( self, conn: sa.Connection, index_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> bool: - qualified_index_name = qualified(conn, index_name, role=role) + qualified_index_name = qualified(conn, index_name, physical_schema=schema_of(conn, schema_tag=schema_tag)) return bool( conn.scalar( sa.select( @@ -209,9 +209,9 @@ def index_exists( ) def drop_index_if_exists( - self, conn: sa.Connection, index_name: str, *, role: Role = Role.PRIMARY + self, conn: sa.Connection, index_name: str, *, schema_tag: str = Role.PRIMARY.value ) -> None: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, role=role)}") + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, physical_schema=schema_of(conn, schema_tag=schema_tag))}") def truncate_table_batch( self, @@ -220,10 +220,10 @@ def truncate_table_batch( *, restart_identities: bool, cascade: bool, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: sql = "TRUNCATE TABLE " + ", ".join( - qualified(conn, name, role=role) for name in table_names + qualified(conn, name, physical_schema=schema_of(conn, schema_tag=schema_tag)) for name in table_names ) if restart_identities: sql += " RESTART IDENTITY" @@ -239,9 +239,9 @@ def find_sequence_name( table_name: str, column_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> str | None: - fully_qualified = qualified(conn, table_name, role=role) + fully_qualified = qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag)) return conn.execute( sa.text("SELECT pg_get_serial_sequence(:table_name, :column_name)"), {"table_name": fully_qualified, "column_name": column_name}, @@ -329,9 +329,9 @@ def install_fulltext_on_table( index_name: str, create_indexes: bool, fastupdate: bool, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: - qualified_table = qualified(conn, table_name, role=role) + qualified_table = qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag)) conn.exec_driver_sql( f"ALTER TABLE {qualified_table} ADD COLUMN IF NOT EXISTS {vector_column_name} tsvector" ) @@ -340,7 +340,7 @@ def install_fulltext_on_table( table_name, sa.MetaData(), sa.Column(vector_column_name, TSVECTOR), - schema=schema_of(conn, role=role), + schema=schema_of(conn, schema_tag=schema_tag), ) index = sa.Index( index_name, @@ -358,13 +358,13 @@ def populate_fulltext_on_table( vector_column_name: str, source_column_name: str, regconfig: str, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> int | None: lightweight_table = sa.table( table_name, sa.column(vector_column_name), sa.column(source_column_name), - schema=schema_of(conn, role=role), + schema=schema_of(conn, schema_tag=schema_tag), ) source_column = lightweight_table.c[source_column_name] stmt = lightweight_table.update().values( @@ -388,12 +388,12 @@ def drop_fulltext_on_table( vector_column_name: str, index_name: str, drop_indexes: bool, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: if drop_indexes: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, role=role)}") + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, physical_schema=schema_of(conn, schema_tag=schema_tag))}") conn.exec_driver_sql( - f"ALTER TABLE {qualified(conn, table_name, role=role)}" + f"ALTER TABLE {qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag))}" f" DROP COLUMN IF EXISTS {vector_column_name}" ) @@ -405,7 +405,7 @@ def prepare_backup( output_path: str, backup_format: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[str, list[str], dict[str, str], str]: tool_path = _pg_dump_path() url = engine.url @@ -424,7 +424,7 @@ def prepare_backup( "--no-owner", "--no-privileges", ] - db_schema = schema_of(engine, role=role) + db_schema = schema_of(engine, schema_tag=schema_tag) if db_schema: command.extend(["--schema", db_schema]) env = os.environ.copy() @@ -438,7 +438,7 @@ def prepare_restore( input_path: str, backup_format: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> tuple[str, list[str], dict[str, str], str]: url = engine.url database_name = url.database @@ -447,7 +447,7 @@ def prepare_restore( "Database restore requires a database name in the configured engine URL." ) connection_uri = _libpq_connection_uri(url) - db_schema = schema_of(engine, role=role) + db_schema = schema_of(engine, schema_tag=schema_tag) if backup_format == "custom": tool_path = _pg_restore_path() diff --git a/omop_alchemy/backends/sqlite.py b/omop_alchemy/backends/sqlite.py index f318062..c7fb640 100644 --- a/omop_alchemy/backends/sqlite.py +++ b/omop_alchemy/backends/sqlite.py @@ -21,7 +21,7 @@ def index_exists( conn: sa.Connection, index_name: str, *, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> bool: row = conn.exec_driver_sql( "SELECT 1 FROM sqlite_master WHERE type='index' AND name=?", @@ -35,7 +35,7 @@ def analyze_table( table_name: str, *, vacuum: bool = False, - role: Role = Role.PRIMARY, + schema_tag: str = Role.PRIMARY.value, ) -> None: if vacuum: raise FeatureNotSupportedError("VACUUM ANALYZE", self) diff --git a/omop_alchemy/cdm/base/decorators.py b/omop_alchemy/cdm/base/decorators.py index b8e6247..b711be0 100644 --- a/omop_alchemy/cdm/base/decorators.py +++ b/omop_alchemy/cdm/base/decorators.py @@ -1,4 +1,7 @@ from typing import TypeVar + +from oa_configurator import validate_schema_tag + from .cdm_table_base import CDMTableBase T = TypeVar("T", bound=type) @@ -18,6 +21,7 @@ def cdm_table(cls: T) -> T: - Forces __abstract__ = False - Ensures __tablename__ is defined - Inherits from CDMTableBase + - Validates the table's own schema tag (see validate_schema_tag) - Used to clearly distinguish real CDM tables from mixins """ @@ -33,6 +37,16 @@ def cdm_table(cls: T) -> T: f"{cls.__name__} must inherit from CDMTableBase " ) + try: + schema_tag = validate_schema_tag(cls.__table__) # ty: ignore[unresolved-attribute] + except ValueError as exc: + raise TypeError(f"@cdm_table on {cls.__name__}: {exc}") from exc + if schema_tag is None: + raise TypeError( + f"@cdm_table on {cls.__name__}: table has no schema tag. " + "Every CDM table must declare __table_args__['schema']." + ) + # Explicitly mark as concrete cls.__abstract__ = False cls.__omop_is_cdm_table__ = True diff --git a/omop_alchemy/config.py b/omop_alchemy/config.py index a36e5b2..5a63545 100644 --- a/omop_alchemy/config.py +++ b/omop_alchemy/config.py @@ -126,7 +126,7 @@ def vocabulary_identity(resolved: ResolvedCDMDatabase) -> str | None: Both fall back to per-engine caching instead of being wrong. """ - vocab_target = resolved.connection_target(Role.VOCAB) + vocab_target = resolved.connection_for_role(Role.VOCAB) if ( vocab_target.safe_url != resolved.connection.safe_url diff --git a/omop_alchemy/maintenance/cli_foreign_keys.py b/omop_alchemy/maintenance/cli_foreign_keys.py index 88ec91a..a977326 100644 --- a/omop_alchemy/maintenance/cli_foreign_keys.py +++ b/omop_alchemy/maintenance/cli_foreign_keys.py @@ -7,7 +7,7 @@ import sqlalchemy as sa import typer -from oa_configurator import Role, schema_of +from oa_configurator import schema_of from ..backends import Backend, resolve_backend, require_backend_support, backend_support_note from ._cli_utils import Status, dry_label, dry_status, omop_command from .tables import ( @@ -33,7 +33,7 @@ class ForeignKeyBase: table_name: str category: TableCategory - role: Role + schema_tag: str @dataclass(frozen=True) @@ -106,7 +106,7 @@ def _collect_fk_info( for table_name in selected_names: foreign_keys = inspector.get_foreign_keys( - table_name, schema=schema_of(engine, role=tables_by_name[table_name].role) + table_name, schema=schema_of(engine, schema_tag=tables_by_name[table_name].schema_tag) ) relevant_foreign_keys = [ foreign_key @@ -131,7 +131,7 @@ def _collect_fk_info( _FKTableInfo( table_name=table.table_name, category=table.category, - role=table.role, + schema_tag=table.schema_tag, outgoing_constraint_count=outgoing_count, incoming_constraint_count=incoming_count, ) @@ -164,9 +164,9 @@ def _collect_strict_validation_failures( } for table_name in sorted(selected_names): - source_role = tables_by_name[table_name].role + source_schema_tag = tables_by_name[table_name].schema_tag for foreign_key in inspector.get_foreign_keys( - table_name, schema=schema_of(connection, role=source_role) + table_name, schema=schema_of(connection, schema_tag=source_schema_tag) ): referred_table = foreign_key.get("referred_table") constrained_columns = foreign_key.get("constrained_columns") or [] @@ -185,8 +185,8 @@ def _collect_strict_validation_failures( str(referred_table), list(constrained_columns), list(referred_columns), - source_role=source_role, - referred_role=tables_by_name[str(referred_table)].role, + source_schema_tag=source_schema_tag, + referred_schema_tag=tables_by_name[str(referred_table)].schema_tag, ) if violation_count == 0: @@ -257,7 +257,7 @@ def validate_foreign_key_constraints( ForeignKeyValidationResult( table_name=target.table_name, category=target.category, - role=target.role, + schema_tag=target.schema_tag, outgoing_constraint_count=target.outgoing_constraint_count, incoming_constraint_count=target.incoming_constraint_count, violating_constraint_count=violating_constraint_count, @@ -313,7 +313,7 @@ def manage_foreign_key_triggers( ForeignKeyManagementResult( table_name=target.table_name, category=target.category, - role=target.role, + schema_tag=target.schema_tag, outgoing_constraint_count=target.outgoing_constraint_count, incoming_constraint_count=target.incoming_constraint_count, enable=enable, @@ -336,14 +336,14 @@ def manage_foreign_key_triggers( for target in targets: if not dry_run: backend.toggle_fk_triggers( - connection, target.table_name, enable=enable, role=target.role + connection, target.table_name, enable=enable, schema_tag=target.schema_tag ) results.append( ForeignKeyManagementResult( table_name=target.table_name, category=target.category, - role=target.role, + schema_tag=target.schema_tag, outgoing_constraint_count=target.outgoing_constraint_count, incoming_constraint_count=target.incoming_constraint_count, enable=enable, @@ -373,13 +373,13 @@ def collect_foreign_key_trigger_status( with engine.connect() as connection: for target in targets: disabled_count, enabled_count = backend.get_fk_trigger_counts( - connection, target.table_name, role=target.role + connection, target.table_name, schema_tag=target.schema_tag ) results.append( ForeignKeyStatusResult( table_name=target.table_name, category=target.category, - role=target.role, + schema_tag=target.schema_tag, disabled_trigger_count=disabled_count, enabled_trigger_count=enabled_count, outgoing_constraint_count=target.outgoing_constraint_count, diff --git a/omop_alchemy/maintenance/cli_fulltext.py b/omop_alchemy/maintenance/cli_fulltext.py index c8f3940..6a358dd 100644 --- a/omop_alchemy/maintenance/cli_fulltext.py +++ b/omop_alchemy/maintenance/cli_fulltext.py @@ -9,7 +9,7 @@ import typer import sqlalchemy as sa from sqlalchemy.engine import Engine -from oa_configurator import Role, role_of_table +from oa_configurator import ResolvedCDMDatabase, Role, guard_schema_provenance_for, validate_schema_tag from ..backends import backend_support_note as _backend_support_note from ..backends import resolve_backend, require_backend_support @@ -29,12 +29,15 @@ } -def _role_for_target(table_name: str) -> Role: - """The Role a fulltext target table's own declared schema tag names. - Resolves via role_of_table() rather than hardcoding Role.VOCAB, +def _schema_tag_for_target(table_name: str) -> str: + """The schema tag a fulltext target table's own declared schema names. + Resolves via validate_schema_tag() rather than hardcoding Role.VOCAB, so a future non-vocab fulltext target resolves correctly. """ - return role_of_table(_FULLTEXT_TARGET_TABLES[table_name]) + tag = validate_schema_tag(_FULLTEXT_TARGET_TABLES[table_name]) + if tag is None: + raise TypeError(f"{table_name}: table has no schema tag.") + return tag app = typer.Typer( @@ -73,6 +76,7 @@ def install_fulltext_columns( create_indexes: bool = True, fastupdate: bool = False, dry_run: bool = False, + resolved: ResolvedCDMDatabase | None = None, ) -> tuple[FullTextResult, ...]: """Install tsvector sidecar columns (and optionally GIN indexes) on OMOP vocabulary tables.""" backend = resolve_backend(engine) @@ -81,7 +85,15 @@ def install_fulltext_columns( try: if not dry_run: + tables_by_schema_tag: dict[str, list[sa.Table]] = {} + for cfg in targets: + tag = _schema_tag_for_target(cfg.table_name) + tables_by_schema_tag.setdefault(tag, []).append(_FULLTEXT_TARGET_TABLES[cfg.table_name]) with engine.begin() as connection: + for schema_tag, tables in tables_by_schema_tag.items(): + # A same-named column/index could already exist under a drifted schema, attached to an unrelated table. + with guard_schema_provenance_for(connection, resolved, role=Role(schema_tag), tables=tables): + pass for cfg in targets: backend.install_fulltext_on_table( connection, @@ -90,7 +102,7 @@ def install_fulltext_columns( index_name=cfg.index_name, create_indexes=create_indexes, fastupdate=fastupdate, - role=_role_for_target(cfg.table_name), + schema_tag=_schema_tag_for_target(cfg.table_name), ) backend.register_fulltext_metadata() except FullTextError: @@ -141,7 +153,7 @@ def populate_fulltext_columns( vector_column_name=cfg.vector_column_name, source_column_name=cfg.source_column_name, regconfig=regconfig, - role=_role_for_target(cfg.table_name), + schema_tag=_schema_tag_for_target(cfg.table_name), ) backend.register_fulltext_metadata() except FullTextError: @@ -188,7 +200,7 @@ def drop_fulltext_columns( vector_column_name=cfg.vector_column_name, index_name=cfg.index_name, drop_indexes=drop_indexes, - role=_role_for_target(cfg.table_name), + schema_tag=_schema_tag_for_target(cfg.table_name), ) backend.unregister_fulltext_metadata() except FullTextError: @@ -243,6 +255,7 @@ def install_fulltext_command( create_indexes=create_indexes, fastupdate=fastupdate, dry_run=dry_run, + resolved=conn.resolved, ) console.print(render_fulltext_results(results)) console.print(render_fulltext_summary(results, action="install", dry_run=dry_run)) diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index 963c8ef..fdd6720 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -10,7 +10,7 @@ from sqlalchemy.exc import DBAPIError, IntegrityError import typer -from oa_configurator import Role, ensure_schema, schema_of, supports_schemas +from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance_for, schema_of, supports_schemas from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY @@ -36,7 +36,7 @@ class IndexTarget: table_name: str category: TableCategory - role: Role + schema_tag: str index_name: str column_names: tuple[str, ...] unique: bool @@ -623,7 +623,7 @@ def collect_index_targets( targets: list[IndexTarget] = [] for table in selected_tables: - table_schema = schema_of(engine, role=table.role) + table_schema = schema_of(engine, schema_tag=table.schema_tag) if not inspector.has_table(table.table_name, schema=table_schema): continue @@ -645,7 +645,7 @@ def collect_index_targets( IndexTarget( table_name=table.table_name, category=table.category, - role=table.role, + schema_tag=table.schema_tag, index_name=physical_name, column_names=column_names, unique=unique, @@ -687,6 +687,7 @@ def manage_indexes( vocabulary_included: bool = False, dry_run: bool = False, cluster: bool = True, + resolved: ResolvedCDMDatabase | None = None, ) -> list[IndexManagementResult]: """Create or drop all ORM-defined indexes. CLUSTERs tables when enabling and cluster=True.""" backend = resolve_backend(engine) @@ -695,10 +696,22 @@ def manage_indexes( metadata_indexes = _schema_metadata_indexes(selected_tables) clustering_supported = backend_supports(backend, "cluster_table") + if enable and not dry_run: + tables_by_schema_tag: dict[str, list[sa.Table]] = {} + for table in selected_tables: + tables_by_schema_tag.setdefault(table.schema_tag, []).append(table.table) + with engine.begin() as guard_connection: + for schema_tag, tables in tables_by_schema_tag.items(): + # A same-named index could already exist under a drifted schema, attached to an unrelated table. + with guard_schema_provenance_for( + guard_connection, resolved, role=Role(schema_tag), tables=tables + ): + pass + results: list[IndexManagementResult] = [] for table in selected_tables: - db_schema = schema_of(engine, role=table.role) + db_schema = schema_of(engine, schema_tag=table.schema_tag) if not inspector.has_table(table.table_name, schema=db_schema): continue @@ -747,7 +760,7 @@ def manage_indexes( if not enable: if not dry_run: existed_before_drop = backend.index_exists( - connection, index_name, role=table.role + connection, index_name, schema_tag=table.schema_tag ) else: existed_before_drop = exists @@ -772,7 +785,7 @@ def manage_indexes( if captured: if not dry_run: backend.drop_index_if_exists( - connection, equivalent_name, role=table.role + connection, equivalent_name, schema_tag=table.schema_tag ) outcome = _IndexOutcome( status=dry_status(dry_run, Status.CAPTURED), @@ -825,7 +838,7 @@ def manage_indexes( physical_name=index_name, ) elif not dry_run: - backend.drop_index_if_exists(connection, index_name, role=table.role) + backend.drop_index_if_exists(connection, index_name, schema_tag=table.schema_tag) # outcome stays default: applied / "metadata-defined index dropped" # dry-run, existed_before_drop True: outcome stays default ("would be dropped") else: @@ -891,7 +904,7 @@ def manage_indexes( operation="index", table_name=table.table_name, category=table.category, - role=table.role, + schema_tag=table.schema_tag, index_name=physical_name, column_names=column_names, unique=unique, @@ -929,7 +942,7 @@ def manage_indexes( operation="cluster", table_name=table.table_name, category=table.category, - role=table.role, + schema_tag=table.schema_tag, index_name=physical_cluster_name, column_names=cluster_columns, unique=False, @@ -947,7 +960,7 @@ def manage_indexes( if not dry_run: with engine.begin() as connection: backend.cluster_table( - connection, table.table_name, physical_cluster_name, role=table.role + connection, table.table_name, physical_cluster_name, schema_tag=table.schema_tag ) clustered_now = True @@ -956,7 +969,7 @@ def manage_indexes( operation="cluster", table_name=table.table_name, category=table.category, - role=table.role, + schema_tag=table.schema_tag, index_name=physical_cluster_name, column_names=cluster_columns, unique=False, @@ -969,7 +982,7 @@ def manage_indexes( if not dry_run and (created_any or clustered_now): with engine.connect() as connection: - backend.analyze_table(connection, table.table_name, role=table.role) + backend.analyze_table(connection, table.table_name, schema_tag=table.schema_tag) connection.commit() return results @@ -1036,6 +1049,7 @@ def enable_indexes_command( vocabulary_included=vocabulary_included, dry_run=dry_run, cluster=cluster, + resolved=conn.resolved, ) console.print(render_index_results(results)) console.print(render_index_summary(results, dry_run=dry_run)) @@ -1073,7 +1087,7 @@ def cluster_tables_command( results: list[IndexManagementResult] = [] for table in selected_tables: - table_schema = schema_of(engine, role=table.role) + table_schema = schema_of(engine, schema_tag=table.schema_tag) if not inspector.has_table(table.table_name, schema=table_schema): continue @@ -1092,10 +1106,10 @@ def cluster_tables_command( if not dry_run: with engine.begin() as connection: backend.cluster_table( - connection, table.table_name, physical_cluster_name, role=table.role + connection, table.table_name, physical_cluster_name, schema_tag=table.schema_tag ) with engine.connect() as connection: - backend.analyze_table(connection, table.table_name, role=table.role) + backend.analyze_table(connection, table.table_name, schema_tag=table.schema_tag) connection.commit() results.append( @@ -1103,7 +1117,7 @@ def cluster_tables_command( operation="cluster", table_name=table.table_name, category=table.category, - role=table.role, + schema_tag=table.schema_tag, index_name=physical_cluster_name, column_names=cluster_columns, unique=False, diff --git a/omop_alchemy/maintenance/cli_schema_reconcile.py b/omop_alchemy/maintenance/cli_schema_reconcile.py index 5274a30..e1d4107 100644 --- a/omop_alchemy/maintenance/cli_schema_reconcile.py +++ b/omop_alchemy/maintenance/cli_schema_reconcile.py @@ -7,10 +7,10 @@ import sqlalchemy as sa from oa_configurator import ( ResolvedDatabase, - Role, find_table_in_other_schemas, - role_of_table, - supports_schemas + schema_of, + supports_schemas, + validate_schema_tag, ) from sqlalchemy.engine.interfaces import ReflectedForeignKeyConstraint, ReflectedIndex @@ -74,26 +74,26 @@ class SchemaReconciliationReport: def _effective_schema( - resolved: ResolvedDatabase | None, role: Role, db_schema: str | None + engine: sa.Engine, + resolved: ResolvedDatabase | None, + schema_tag: str | None, + db_schema: str | None, ) -> str | None: - """resolved.schema_for_role(role) when given, else db_schema regardless - of role, the fallback for a caller with no resolved object to hand. + """schema_of(engine, schema_tag=schema_tag) when resolved is given, else db_schema. + + Uses schema_of against engine to accommodate bare schema_tags. """ - return resolved.schema_for_role(role) if resolved is not None else db_schema + if resolved is None: + return db_schema + return schema_of(engine, schema_tag=schema_tag) def _schema_qualified_tables( - resolved: ResolvedDatabase | None, db_schema: str | None + engine: sa.Engine, resolved: ResolvedDatabase | None, db_schema: str | None ) -> dict[int, sa.Table]: """Schema-qualified copy of every table in Base.metadata, keyed by id() of the original. - Each table is qualified to its own role's schema via resolved, not one - blanket value, since a vocab-role table can live in a different physical - schema than a clinical one. Copied together into one MetaData() (not per - table): to_metadata() never brings a referenced table along on its own, - and an FK's target needs its copy already present in the same metadata. - Returns the tables unchanged, keyed by their own id, when resolved and - db_schema are both None. + Copied into one shared MetaData() since to_metadata() won't bring a referenced table's copy along on its own, which FK targets need present. """ from orm_loader.helpers import Base @@ -102,8 +102,10 @@ def _schema_qualified_tables( metadata = sa.MetaData() def _referred_schema(_table: sa.Table, _to_schema, _constraint, referred_schema: str | None): - # None means "unchanged" to to_metadata(); BLANK_SCHEMA is what actually clears a schema tag. - target = _effective_schema(resolved, Role(referred_schema), db_schema) + # to_metadata(): None means "unchanged", BLANK_SCHEMA actually clears the schema. + # referred_schema is already validate_schema_tag()-clean (checked below), no + # re-validation needed here. + target = _effective_schema(engine, resolved, referred_schema, db_schema) return target if target is not None else sa.BLANK_SCHEMA return { @@ -111,7 +113,7 @@ def _referred_schema(_table: sa.Table, _to_schema, _constraint, referred_schema: metadata, # SQLAlchemy's own stub omits None from schema's declared type, # despite accepting and correctly handling it at runtime - schema=_effective_schema(resolved, role_of_table(table), db_schema), # ty: ignore[invalid-argument-type] + schema=_effective_schema(engine, resolved, validate_schema_tag(table), db_schema), # ty: ignore[invalid-argument-type] referred_schema_fn=_referred_schema, ) for table in Base.metadata.tables.values() @@ -221,9 +223,9 @@ def reconcile_schema( Engine to inspect. Its dialect selects the backend used for cluster-state checks. resolved : ResolvedDatabase, optional - When given, qualifies each table to its own role's schema + When given, qualifies each table to its own schema tag (schema_name/vocab_schema/results_schema) instead of applying - db_schema to every table regardless of role. + db_schema to every table regardless of tag. db_schema : str, optional Blanket schema applied to every table when resolved is not given. vocabulary_included : bool, optional @@ -241,7 +243,7 @@ def reconcile_schema( _backend = resolve_backend(engine) _cross_schema_fk_supported = supports_schemas(engine) selected_tables = select_maintenance_tables(exclude_categories=excluded_categories) - schema_qualified_tables = _schema_qualified_tables(resolved, db_schema) + schema_qualified_tables = _schema_qualified_tables(engine, resolved, db_schema) inspector = sa.inspect(engine) all_issues: list[ReconciliationIssue] = [] table_results: list[TableReconciliationResult] = [] @@ -249,8 +251,10 @@ def reconcile_schema( with engine.connect() as connection: for maintenance_table in selected_tables: table_issues: list[ReconciliationIssue] = [] - table_role = role_of_table(maintenance_table.table) - table_schema = _effective_schema(resolved, table_role, db_schema) + table_schema_tag = validate_schema_tag(maintenance_table.table) + if table_schema_tag is None: + raise TypeError(f"{maintenance_table.table_name}: table has no schema tag.") + table_schema = _effective_schema(engine, resolved, table_schema_tag, db_schema) exists = inspector.has_table(maintenance_table.table_name, schema=table_schema) if not exists: relocated_to = find_table_in_other_schemas( @@ -400,8 +404,8 @@ def reconcile_schema( expected_fks = _expected_foreign_keys(expected_table) actual_fks = _actual_foreign_keys(inspector, maintenance_table.table_name, table_schema) - # Uses the unqualified table, since two role-tagged tables can collapse to - # the same schema (e.g. all-None on SQLite) and hide a genuine cross-role FK. + # Uses the unqualified table, since two differently-tagged tables can collapse to + # the same schema (e.g. all-None on SQLite) and hide a genuine cross-tag FK. raw_expected_fks = _expected_foreign_keys(maintenance_table.table) for signature, constraint in expected_fks.items(): @@ -410,7 +414,7 @@ def reconcile_schema( if ( not _cross_schema_fk_supported and raw_constraint is not None - and role_of_table(raw_constraint.referred_table) != table_role + and validate_schema_tag(raw_constraint.referred_table) != table_schema_tag ): # SQLite can never create an inline FK crossing a schema boundary. continue @@ -522,7 +526,7 @@ def reconcile_schema( actual_cluster = _backend.get_clustered_index_name( connection, maintenance_table.table_name, - role=table_role, + schema_tag=table_schema_tag, ) if expected_cluster != actual_cluster: # May be a rename, not drift, so treat like a renamed index. diff --git a/omop_alchemy/maintenance/cli_schema_summary.py b/omop_alchemy/maintenance/cli_schema_summary.py index a348de9..4dc5ad0 100644 --- a/omop_alchemy/maintenance/cli_schema_summary.py +++ b/omop_alchemy/maintenance/cli_schema_summary.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from oa_configurator import Role, qualified, schema_of +from oa_configurator import qualified, schema_of from .tables import TableCategory, select_omop_tables @@ -16,7 +16,7 @@ class TableSummaryResult: table_name: str category: TableCategory - role: Role + schema_tag: str model_name: str primary_key_columns: tuple[str, ...] exists: bool @@ -36,7 +36,7 @@ def collect_data_summary( results: list[TableSummaryResult] = [] with engine.connect() as connection: for table in tables: - exists = inspector.has_table(table.table_name, schema=schema_of(engine, role=table.role)) + exists = inspector.has_table(table.table_name, schema=schema_of(engine, schema_tag=table.schema_tag)) if not exists and existing_only: continue @@ -45,7 +45,7 @@ def collect_data_summary( row_count = int( connection.execute( sa.text( - f"SELECT COUNT(*) FROM {qualified(connection, table.table_name, role=table.role)}" + f"SELECT COUNT(*) FROM {qualified(connection, table.table_name, physical_schema=schema_of(connection, schema_tag=table.schema_tag))}" ) ).scalar_one() ) @@ -54,7 +54,7 @@ def collect_data_summary( TableSummaryResult( table_name=table.table_name, category=table.category, - role=table.role, + schema_tag=table.schema_tag, model_name=table.model_name, primary_key_columns=table.primary_key_names, exists=exists, diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index be127b3..521db23 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance, schema_of +from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance_for, schema_of from orm_loader.helpers import Base from ._cli_utils import Status, dry_label, dry_status from .tables import ( @@ -76,14 +76,8 @@ def create_missing_tables( """ vocab_engine = vocab_engine if vocab_engine is not None else engine if not dry_run: - ensure_schema(engine, schema_of(engine, role=Role.PRIMARY)) - # Primary alone isn't enough: on a genuinely fresh database, - # vocab_schema/results_schema don't exist yet either -- without this, - # create_all() below fails "schema does not exist" for every - # vocab/results table instead of creating it. ensure_schema() itself - # already no-ops for None/already-default/schema-incapable dialects, - # so calling it for RESULTS (which usually equals the primary schema) - # is safe. + ensure_schema(engine, schema_of(engine, schema_tag=Role.PRIMARY)) + # create_all() would fail for non-existing vocab/results schemas on a fresh database if resolved is not None: ensure_schema(engine, resolved.schema_for_role(Role.RESULTS)) ensure_schema(vocab_engine, resolved.schema_for_role(Role.VOCAB)) @@ -92,14 +86,10 @@ def create_missing_tables( engine, vocabulary_included=vocabulary_included, ) - # Each role's own schema, not just primary: a dependency table can be - # vocab- or results-role, and checking only the primary schema here used - # to make every already-existing vocab/results table look absent to the - # dependency-resolution pass below, incorrectly blocking creation of - # clinical tables that reference them. + # Checking only primary schema would hide existing vocab/results tables, wrongly blocking dependents. existing_table_names: set[str] = set() for role in Role: - existing_table_names |= set(inspector.get_table_names(schema=schema_of(engine, role=role))) + existing_table_names |= set(inspector.get_table_names(schema=schema_of(engine, schema_tag=role))) missing_table_names = {table.table_name for table in missing_tables} blocked_dependencies: dict[str, tuple[str, ...]] = {} @@ -122,13 +112,17 @@ def create_missing_tables( results: list[TableCreationResult] = [] if creatable_tables and not dry_run: all_tables = [table.table for table in creatable_tables] + tables_by_role = { + role: [table for table in all_tables if table.schema == role.value] + for role in Role + } if vocab_engine is engine: # One call: create_all's dependency sort and FK-deferral must see every table together. with ( engine.begin() as connection, - guard_schema_provenance(connection, resolved, role=Role.PRIMARY), - guard_schema_provenance(connection, resolved, role=Role.RESULTS), - guard_schema_provenance(connection, resolved, role=Role.VOCAB), + guard_schema_provenance_for(connection, resolved, role=Role.PRIMARY, tables=tables_by_role[Role.PRIMARY]), + guard_schema_provenance_for(connection, resolved, role=Role.RESULTS, tables=tables_by_role[Role.RESULTS]), + guard_schema_provenance_for(connection, resolved, role=Role.VOCAB, tables=tables_by_role[Role.VOCAB]), ): Base.metadata.create_all( bind=connection, tables=all_tables, checkfirst=True @@ -136,17 +130,13 @@ def create_missing_tables( else: # Split physical connections: a cross-boundary FK can't be created here at all; # that failure surfaces from create_all itself rather than being masked. - vocab_tables = [ - table for table in all_tables if table.schema == Role.VOCAB.value - ] - other_tables = [ - table for table in all_tables if table.schema != Role.VOCAB.value - ] + vocab_tables = tables_by_role[Role.VOCAB] + other_tables = tables_by_role[Role.PRIMARY] + tables_by_role[Role.RESULTS] if other_tables: with ( engine.begin() as connection, - guard_schema_provenance(connection, resolved, role=Role.PRIMARY), - guard_schema_provenance(connection, resolved, role=Role.RESULTS), + guard_schema_provenance_for(connection, resolved, role=Role.PRIMARY, tables=tables_by_role[Role.PRIMARY]), + guard_schema_provenance_for(connection, resolved, role=Role.RESULTS, tables=tables_by_role[Role.RESULTS]), ): Base.metadata.create_all( bind=connection, tables=other_tables, checkfirst=True @@ -154,7 +144,7 @@ def create_missing_tables( if vocab_tables: with ( vocab_engine.begin() as vocab_connection, - guard_schema_provenance(vocab_connection, resolved, role=Role.VOCAB), + guard_schema_provenance_for(vocab_connection, resolved, role=Role.VOCAB, tables=vocab_tables), ): Base.metadata.create_all( bind=vocab_connection, tables=vocab_tables, checkfirst=True diff --git a/omop_alchemy/maintenance/cli_tables.py b/omop_alchemy/maintenance/cli_tables.py index f4aa752..55dcc69 100644 --- a/omop_alchemy/maintenance/cli_tables.py +++ b/omop_alchemy/maintenance/cli_tables.py @@ -7,7 +7,7 @@ import sqlalchemy as sa import typer -from oa_configurator import Role, autocommit_connection, qualified, schema_of +from oa_configurator import ResolvedCDMDatabase, Role, autocommit_connection, guard_schema_provenance_for, qualified, schema_of from ..backends import resolve_backend, require_backend_support, backend_support_note from ._cli_utils import Status, dry_label, dry_status, omop_command, resolve_selection from .tables import ( @@ -39,7 +39,7 @@ class AnalyzeTableResult: table_name: str category: TableCategory - role: Role + schema_tag: str operation: str status: Status detail: str @@ -70,13 +70,13 @@ def analyze_tables( with connection_factory as connection: for maintenance_table in selected_tables: - table_schema = schema_of(engine, role=maintenance_table.role) + table_schema = schema_of(engine, schema_tag=maintenance_table.schema_tag) if not inspector.has_table(maintenance_table.table_name, schema=table_schema): results.append( AnalyzeTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, - role=maintenance_table.role, + schema_tag=maintenance_table.schema_tag, operation=operation, status=Status.SKIPPED, detail="table not present in target database", @@ -86,14 +86,14 @@ def analyze_tables( if not dry_run: backend.analyze_table( - connection, maintenance_table.table_name, vacuum=vacuum, role=maintenance_table.role + connection, maintenance_table.table_name, vacuum=vacuum, schema_tag=maintenance_table.schema_tag ) results.append( AnalyzeTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, - role=maintenance_table.role, + schema_tag=maintenance_table.schema_tag, operation=operation, status=dry_status(dry_run), detail=dry_label(dry_run, f"{operation.lower()} would run", f"{operation.lower()} completed"), @@ -113,7 +113,7 @@ class TruncateTableResult: table_name: str category: TableCategory - role: Role + schema_tag: str row_count: int | None status: Status detail: str @@ -134,7 +134,7 @@ def _blocking_foreign_key_references( blockers: dict[str, set[str]] = {} for role in Role: - role_schema = schema_of(engine, role=role) + role_schema = schema_of(engine, schema_tag=role) for table_name in inspector.get_table_names(schema=role_schema): if table_name in selected_table_names: continue @@ -173,6 +173,7 @@ def truncate_tables( restart_identities: bool = False, cascade: bool = False, dry_run: bool = False, + resolved: ResolvedCDMDatabase | None = None, ) -> list[TruncateTableResult]: """Truncate selected ORM-managed tables. Raises if non-selected tables hold blocking FK references.""" if scope is not None and table_names is not None: @@ -183,21 +184,22 @@ def truncate_tables( backend = resolve_backend(engine) require_backend_support(backend, "truncate_table_batch", "Table truncation") selected_tables = resolve_maintenance_tables(scope=scope, table_names=table_names) + tables_by_name = {table.table_name: table for table in selected_tables} inspector = sa.inspect(engine) results: list[TruncateTableResult] = [] existing_tables: list[str] = [] - existing_table_names_by_role: dict[Role, list[str]] = {} + existing_table_names_by_schema_tag: dict[str, list[str]] = {} with engine.begin() as connection: for maintenance_table in selected_tables: if not inspector.has_table( - maintenance_table.table_name, schema=schema_of(engine, role=maintenance_table.role) + maintenance_table.table_name, schema=schema_of(engine, schema_tag=maintenance_table.schema_tag) ): results.append( TruncateTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, - role=maintenance_table.role, + schema_tag=maintenance_table.schema_tag, row_count=None, status=Status.SKIPPED, detail="table not present in target database", @@ -207,18 +209,18 @@ def truncate_tables( row_count = int( connection.exec_driver_sql( - f"SELECT COUNT(*) FROM {qualified(connection, maintenance_table.table_name, role=maintenance_table.role)}" + f"SELECT COUNT(*) FROM {qualified(connection, maintenance_table.table_name, physical_schema=schema_of(connection, schema_tag=maintenance_table.schema_tag))}" ).scalar_one() ) existing_tables.append(maintenance_table.table_name) - existing_table_names_by_role.setdefault(maintenance_table.role, []).append( + existing_table_names_by_schema_tag.setdefault(maintenance_table.schema_tag, []).append( maintenance_table.table_name ) results.append( TruncateTableResult( table_name=maintenance_table.table_name, category=maintenance_table.category, - role=maintenance_table.role, + schema_tag=maintenance_table.schema_tag, row_count=row_count, status=dry_status(dry_run), detail=dry_label(dry_run, "table would be truncated", "table truncated"), @@ -235,18 +237,26 @@ def truncate_tables( raise RuntimeError(_format_blocking_reference_error(blockers)) if existing_tables and not dry_run: - # One TRUNCATE per role: truncate_table_batch qualifies every name - # in its list with a single role, so a selection spanning more - # than one role (category and role are independent axes -- see - # MaintenanceTable.role) is split into one batch per role rather - # than misqualifying some of the names. - for role, table_names_for_role in existing_table_names_by_role.items(): + # One TRUNCATE per schema tag: truncate_table_batch qualifies every + # name in its list with a single tag, so a selection spanning more + # than one tag (category and schema_tag are independent axes -- + # see MaintenanceTable.schema_tag) is split into one batch per tag + # rather than misqualifying some of the names. + for schema_tag, table_names_for_tag in existing_table_names_by_schema_tag.items(): + # A same-named table could already exist under a drifted schema, so truncate could hit unrelated data. + with guard_schema_provenance_for( + connection, + resolved, + role=Role(schema_tag), + tables=[tables_by_name[name].table for name in table_names_for_tag], + ): + pass backend.truncate_table_batch( connection, - table_names_for_role, + table_names_for_tag, restart_identities=restart_identities, cascade=cascade, - role=role, + schema_tag=schema_tag, ) return results @@ -262,7 +272,7 @@ class SequenceTarget: table_name: str category: TableCategory - role: Role + schema_tag: str pk_column_name: str @@ -272,7 +282,7 @@ class SequenceResetResult: table_name: str category: TableCategory - role: Role + schema_tag: str pk_column_name: str sequence_name: str | None next_value: int | None @@ -297,7 +307,7 @@ def collect_sequence_targets( SequenceTarget( table_name=table.table_name, category=table.category, - role=table.role, + schema_tag=table.schema_tag, pk_column_name=pk_column_name, ) ) @@ -319,11 +329,11 @@ def reset_model_sequences( with engine.begin() as connection: for target in targets: - if not inspector.has_table(target.table_name, schema=schema_of(engine, role=target.role)): + if not inspector.has_table(target.table_name, schema=schema_of(engine, schema_tag=target.schema_tag)): continue sequence_name = backend.find_sequence_name( - connection, target.table_name, target.pk_column_name, role=target.role + connection, target.table_name, target.pk_column_name, schema_tag=target.schema_tag ) if sequence_name is None: @@ -331,7 +341,7 @@ def reset_model_sequences( SequenceResetResult( table_name=target.table_name, category=target.category, - role=target.role, + schema_tag=target.schema_tag, pk_column_name=target.pk_column_name, sequence_name=None, next_value=None, @@ -341,7 +351,9 @@ def reset_model_sequences( ) continue - fully_qualified = qualified(connection, target.table_name, role=target.role) + fully_qualified = qualified( + connection, target.table_name, physical_schema=schema_of(connection, schema_tag=target.schema_tag) + ) current_max = connection.execute( sa.text( f"SELECT COALESCE(MAX({target.pk_column_name}), 0) " @@ -357,7 +369,7 @@ def reset_model_sequences( SequenceResetResult( table_name=target.table_name, category=target.category, - role=target.role, + schema_tag=target.schema_tag, pk_column_name=target.pk_column_name, sequence_name=sequence_name, next_value=next_value, @@ -495,6 +507,7 @@ def truncate_tables_command( restart_identities=restart_identities, cascade=cascade, dry_run=dry_run, + resolved=conn.resolved, ) console.print(render_truncate_results(results)) console.print( diff --git a/omop_alchemy/maintenance/cli_vocab.py b/omop_alchemy/maintenance/cli_vocab.py index b1fcf92..7604ccf 100644 --- a/omop_alchemy/maintenance/cli_vocab.py +++ b/omop_alchemy/maintenance/cli_vocab.py @@ -298,9 +298,9 @@ def load_vocab_source( Engine to create vocabulary tables on, when ``vocab_connection`` names a physically different server than ``engine``. Defaults to ``engine``. CSV loading itself still runs entirely against ``engine``; this only - affects where vocab-role tables get created. + affects where vocab-tagged tables get created. vocab_schema : str, optional - Schema vocab-role tables live in, for the table-existence check + Schema vocab-tagged tables live in, for the table-existence check against ``vocab_engine``. Defaults to ``db_schema``. """ vocab_engine = vocab_engine if vocab_engine is not None else engine diff --git a/omop_alchemy/maintenance/tables.py b/omop_alchemy/maintenance/tables.py index 1a0ff11..ce13192 100644 --- a/omop_alchemy/maintenance/tables.py +++ b/omop_alchemy/maintenance/tables.py @@ -5,7 +5,7 @@ from typing import Iterable import sqlalchemy as sa -from oa_configurator import Role, role_of_table, schema_of +from oa_configurator import schema_of, validate_schema_tag class TableCategory(StrEnum): @@ -58,16 +58,19 @@ class MaintenanceTable: primary_key_columns: tuple[sa.Column[object], ...] @property - def role(self) -> Role: - """The schema_translate_map role this table's data physically lives - under (oa_configurator.Role), read off its own declared schema tag. + def schema_tag(self) -> str: + """The schema_translate_map key this table's data physically lives + under, read off its own declared schema tag. Independent of TableCategory: category is a logical/folder grouping (e.g. cohort/cohort_definition are RESULTS-category despite - classifying as "derived" in the CDM sense), role is where the + classifying as "derived" in the CDM sense), schema_tag is where the table's rows physically live. """ - return role_of_table(self.table) + tag = validate_schema_tag(self.table) + if tag is None: + raise TypeError(f"{self.table_name}: table has no schema tag.") + return tag @property def is_vocabulary(self) -> bool: @@ -252,15 +255,15 @@ def existing_maintenance_tables( vocabulary_included: bool, require_single_integer_primary_key: bool = False, ) -> list[MaintenanceTable]: - """ORM-managed tables that already exist, each checked against its own role's schema. + """ORM-managed tables that already exist, each checked against its own schema tag. Parameters ---------- bindable : sqlalchemy.Engine or sqlalchemy.Connection Used both to inspect the database and, via its schema_translate_map, - to resolve each table's own role to a physical schema - (``schema_of(bindable, role=table.role)``) -- a blanket schema - passed in once would silently misclassify every vocab/results + to resolve each table's own schema tag to a physical schema + (``schema_of(bindable, schema_tag=table.schema_tag)``) -- a blanket + schema passed in once would silently misclassify every vocab/results table checked against a database with a genuine primary/vocab/ results split. """ @@ -271,7 +274,7 @@ def existing_maintenance_tables( vocabulary_included=vocabulary_included, require_single_integer_primary_key=require_single_integer_primary_key, ) - if inspector.has_table(table.table_name, schema=schema_of(bindable, role=table.role)) + if inspector.has_table(table.table_name, schema=schema_of(bindable, schema_tag=table.schema_tag)) ] @@ -280,7 +283,7 @@ def missing_maintenance_tables( *, vocabulary_included: bool, ) -> list[MaintenanceTable]: - """ORM-managed tables that are absent, each checked against its own role's schema. + """ORM-managed tables that are absent, each checked against its own schema tag. See :func:`existing_maintenance_tables` for why *bindable* replaces a single ``db_schema`` string. @@ -289,5 +292,5 @@ def missing_maintenance_tables( return [ table for table in select_omop_tables(vocabulary_included=vocabulary_included) - if not inspector.has_table(table.table_name, schema=schema_of(bindable, role=table.role)) + if not inspector.has_table(table.table_name, schema=schema_of(bindable, schema_tag=table.schema_tag)) ] diff --git a/tests/conftest.py b/tests/conftest.py index 3323bf6..1539e7a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -71,7 +71,7 @@ def resolved_cdm_database_from_engine( def fresh_engine() -> Iterator[sa.Engine]: """Fresh, empty, function-scoped SQLite engine. - SQLite has no schema concept, so every role maps back to None, matching + SQLite has no schema concept, so every schema tag maps back to None, matching the flat namespace every caller here has always assumed. """ with isolated_test_database( @@ -370,9 +370,9 @@ def engine(tmp_path_factory: pytest.TempPathFactory) -> Iterator[sa.Engine]: poolclass=sa.pool.StaticPool, connect_args={"check_same_thread": False, "timeout": 30}, # SQLite has no schema concept, and this fixture always represented - # a single flat namespace: map every role back to None so the + # a single flat namespace: map every schema tag back to None so the # vocab/results-tagged tables land in the same place they always - # have here, unaffected by schema role tagging. + # have here, unaffected by schema tagging. execution_options={SCHEMA_TRANSLATE_MAP_KEY: {Role.PRIMARY.value: None, "vocab": None, "results": None}}, ) as db: engine = db.connection.engine @@ -418,7 +418,7 @@ def pg_engine(pg_db): genuine engine-building code paths against (``.connect()``/``.begin()``, which a bare ``Connection`` can't stand in for). - A thin shim over ``pg_db``'s own ``committing_engine``: every role + A thin shim over ``pg_db``'s own ``committing_engine``: every schema tag (``None``, ``"vocab"``, ``"results"``) folds back to the connection's default, matching the single-schema setup ``pg_session`` provides. """ diff --git a/tests/test_foreign_keys.py b/tests/test_foreign_keys.py index 1ae731f..3dcaeeb 100644 --- a/tests/test_foreign_keys.py +++ b/tests/test_foreign_keys.py @@ -100,10 +100,10 @@ def name(self) -> str: def dialect(self) -> str: return "postgresql" - def analyze_table(self, conn, table_name, *, vacuum=False, role=None) -> None: + def analyze_table(self, conn, table_name, *, vacuum=False, schema_tag=None) -> None: pass - def toggle_fk_triggers(self, conn, table_name, *, enable: bool, role=None) -> None: + def toggle_fk_triggers(self, conn, table_name, *, enable: bool, schema_tag=None) -> None: action = "ENABLE" if enable else "DISABLE" conn.exec_driver_sql(f"ALTER TABLE {table_name} {action} TRIGGER ALL") @@ -142,7 +142,7 @@ def begin(self): type("Target", (), { "table_name": "person", "category": "clinical", - "role": Role.PRIMARY, + "schema_tag": Role.PRIMARY.value, "model_name": "Person", "model_module": "omop_alchemy.cdm.model.clinical.person", "outgoing_constraint_count": 1, @@ -151,7 +151,7 @@ def begin(self): type("Target", (), { "table_name": "visit_occurrence", "category": "health_system", - "role": Role.PRIMARY, + "schema_tag": Role.PRIMARY.value, "model_name": "VisitOccurrence", "model_module": "omop_alchemy.cdm.model.health_system.visit_occurrence", "outgoing_constraint_count": 2, @@ -214,7 +214,7 @@ def begin(self): type("Target", (), { "table_name": "person", "category": "clinical", - "role": Role.PRIMARY, + "schema_tag": Role.PRIMARY.value, "model_name": "Person", "model_module": "omop_alchemy.cdm.model.clinical.person", "outgoing_constraint_count": 1, @@ -312,7 +312,7 @@ def connect(self): type("Target", (), { "table_name": "person", "category": "clinical", - "role": Role.PRIMARY, + "schema_tag": Role.PRIMARY.value, "model_name": "Person", "model_module": "omop_alchemy.cdm.model.clinical.person", "outgoing_constraint_count": 1, @@ -321,7 +321,7 @@ def connect(self): type("Target", (), { "table_name": "visit_occurrence", "category": "health_system", - "role": Role.PRIMARY, + "schema_tag": Role.PRIMARY.value, "model_name": "VisitOccurrence", "model_module": "omop_alchemy.cdm.model.health_system.visit_occurrence", "outgoing_constraint_count": 2, @@ -391,7 +391,7 @@ def fake_validate_foreign_key_constraints( ForeignKeyValidationResult( table_name="visit_occurrence", category=TableCategory.HEALTH_SYSTEM, - role=Role.PRIMARY, + schema_tag=Role.PRIMARY.value, outgoing_constraint_count=2, incoming_constraint_count=0, violating_constraint_count=1, diff --git a/tests/test_fulltext.py b/tests/test_fulltext.py index 41e84e7..4036e24 100644 --- a/tests/test_fulltext.py +++ b/tests/test_fulltext.py @@ -232,6 +232,7 @@ def fake_install_fulltext_columns( create_indexes: bool = True, fastupdate: bool = False, dry_run: bool = False, + resolved: object = None, ): calls["engine"] = engine calls["create_indexes"] = create_indexes diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 440350d..c8c0568 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -61,7 +61,7 @@ def indexed_engine(request): bookkeeping_schema = get_bookkeeping_schema(engine) inspector = sa.inspect(engine) if inspector.has_table(_DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema): - table_ref = qualified(engine, _DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema) + table_ref = qualified(engine, _DROPPED_INDEXES_TABLE_NAME, physical_schema=bookkeeping_schema) with engine.begin() as connection: connection.exec_driver_sql(f"DROP TABLE {table_ref}") else: @@ -201,9 +201,9 @@ def test_manage_indexes_enable_analyzes_tables_with_new_indexes(sqlite_indexed_e analyzed_tables: list[str] = [] original_analyze = SQLiteBackend.analyze_table - def recording_analyze(self, conn, table_name, *, vacuum=False, role=Role.PRIMARY): + def recording_analyze(self, conn, table_name, *, vacuum=False, schema_tag=Role.PRIMARY.value): analyzed_tables.append(table_name) - return original_analyze(self, conn, table_name, vacuum=vacuum, role=role) + return original_analyze(self, conn, table_name, vacuum=vacuum, schema_tag=schema_tag) monkeypatch.setattr(SQLiteBackend, "analyze_table", recording_analyze) @@ -345,10 +345,10 @@ def test_manage_indexes_enable_clusters_then_analyzes(sqlite_indexed_engine, mon calls: list[str] = [] - def fake_cluster_table(self, conn, table_name, index_name, *, role=Role.PRIMARY): + def fake_cluster_table(self, conn, table_name, index_name, *, schema_tag=Role.PRIMARY.value): calls.append(f"cluster:{table_name}") - def fake_analyze_table(self, conn, table_name, *, vacuum=False, role=Role.PRIMARY): + def fake_analyze_table(self, conn, table_name, *, vacuum=False, schema_tag=Role.PRIMARY.value): calls.append(f"analyze:{table_name}") monkeypatch.setattr(SQLiteBackend, "cluster_table", fake_cluster_table) @@ -395,7 +395,7 @@ def fake_manage_indexes( operation="index", table_name="person", category=TableCategory.CLINICAL, - role=Role.PRIMARY, + schema_tag=Role.PRIMARY.value, index_name=PERSON_GENDER_INDEX, column_names=("gender_concept_id",), unique=False, @@ -450,6 +450,7 @@ def fake_manage_indexes( vocabulary_included: bool = False, dry_run: bool = False, cluster: bool = True, + resolved: object = None, ) -> list[IndexManagementResult]: calls["enable"] = enable calls["vocabulary_included"] = vocabulary_included @@ -460,7 +461,7 @@ def fake_manage_indexes( operation="index", table_name="person", category=TableCategory.CLINICAL, - role=Role.PRIMARY, + schema_tag=Role.PRIMARY.value, index_name=PERSON_GENDER_INDEX, column_names=("gender_concept_id",), unique=False, @@ -761,14 +762,14 @@ def test_manage_indexes_enable_cluster_uses_restored_physical_name(sqlite_indexe calls: list[tuple[str, str]] = [] - def fake_cluster_table(self, conn, table_name, index_name, *, role=Role.PRIMARY): + def fake_cluster_table(self, conn, table_name, index_name, *, schema_tag=Role.PRIMARY.value): calls.append((table_name, index_name)) monkeypatch.setattr(SQLiteBackend, "cluster_table", fake_cluster_table) monkeypatch.setattr( SQLiteBackend, "analyze_table", - lambda self, conn, table_name, *, vacuum=False, role=Role.PRIMARY: None, + lambda self, conn, table_name, *, vacuum=False, schema_tag=Role.PRIMARY.value: None, ) manage_indexes(engine, enable=True, cluster=True) @@ -823,7 +824,7 @@ def _dropped_indexes_rows(engine: sa.Engine) -> list[dict[str, object]]: inspector = sa.inspect(engine) if not inspector.has_table(_DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema): return [] - table_ref = qualified(engine, _DROPPED_INDEXES_TABLE_NAME, schema=bookkeeping_schema) + table_ref = qualified(engine, _DROPPED_INDEXES_TABLE_NAME, physical_schema=bookkeeping_schema) with engine.connect() as connection: rows = connection.exec_driver_sql( f"SELECT table_name, index_name FROM {table_ref}" @@ -845,7 +846,7 @@ def _warning_result() -> IndexManagementResult: operation="index", table_name="person", category=TableCategory.CLINICAL, - role=Role.PRIMARY, + schema_tag=Role.PRIMARY.value, index_name="idx_gender_partial", column_names=("gender_concept_id",), unique=False, @@ -867,7 +868,7 @@ def test_render_index_summary_omits_warnings_row_when_none(): operation="index", table_name="person", category=TableCategory.CLINICAL, - role=Role.PRIMARY, + schema_tag=Role.PRIMARY.value, index_name=PERSON_GENDER_INDEX, column_names=("gender_concept_id",), unique=False, diff --git a/tests/test_load_vocab_source.py b/tests/test_load_vocab_source.py index d92a8be..0c76cf7 100644 --- a/tests/test_load_vocab_source.py +++ b/tests/test_load_vocab_source.py @@ -698,7 +698,7 @@ def fake_manage_indexes(engine, *, enable, **kwargs): operation="index", table_name="concept", category=TableCategory.VOCABULARY, - role=Role.VOCAB, + schema_tag=Role.VOCAB.value, index_name="idx_concept_partial", column_names=("domain_id",), unique=False, @@ -714,7 +714,7 @@ def fake_manage_indexes(engine, *, enable, **kwargs): operation="index", table_name="concept", category=TableCategory.VOCABULARY, - role=Role.VOCAB, + schema_tag=Role.VOCAB.value, index_name="ix_concept_domain_id", column_names=("domain_id",), unique=False, diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index 5216664..ef20b2f 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -106,9 +106,9 @@ def test_reconcile_schema_reports_no_drift_on_fresh_database(reconcile_engine): def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(reconcile_engine): engine, resolved = reconcile_engine with engine.begin() as connection: - connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX)}") + connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX, physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))}") connection.exec_driver_sql( - f"CREATE INDEX idx_gender ON {qualified(connection, 'person')} (gender_concept_id)" + f"CREATE INDEX idx_gender ON {qualified(connection, 'person', physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))} (gender_concept_id)" ) report = reconcile_schema(engine, resolved=resolved) @@ -124,9 +124,9 @@ def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(rec def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(reconcile_engine): engine, resolved = reconcile_engine with engine.begin() as connection: - connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX)}") + connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX, physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))}") connection.exec_driver_sql( - f"CREATE INDEX idx_gender ON {qualified(connection, 'person')} (gender_concept_id)" + f"CREATE INDEX idx_gender ON {qualified(connection, 'person', physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))} (gender_concept_id)" ) report = reconcile_schema(engine, resolved=resolved) @@ -242,9 +242,9 @@ def _index_issues(report): assert _index_issues(report) == [] with engine.begin() as connection: - connection.exec_driver_sql(f'DROP INDEX {qualified(connection, "ix_concept_concept_name_lower")}') + connection.exec_driver_sql(f'DROP INDEX {qualified(connection, "ix_concept_concept_name_lower", physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))}') connection.exec_driver_sql( - f'CREATE INDEX ix_concept_concept_name_lower ON {qualified(connection, "concept")} (upper(concept_name))' + f'CREATE INDEX ix_concept_concept_name_lower ON {qualified(connection, "concept", physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))} (upper(concept_name))' ) report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) @@ -284,7 +284,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_foreign_cluster_inde monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name, role=None: ( + lambda self, conn, table_name, schema_tag=None: ( "idx_episode_person" if table_name == "episode" else None ), ) @@ -311,7 +311,7 @@ def test_reconcile_schema_cluster_check_still_reports_real_mismatch(fresh_reconc monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name, role=None: ( + lambda self, conn, table_name, schema_tag=None: ( "some_unrelated_index" if table_name == "episode" else None ), ) @@ -345,7 +345,7 @@ def test_reconcile_schema_cluster_check_reports_renamed_for_pk_based_cluster_tar monkeypatch.setattr( SQLiteBackend, "get_clustered_index_name", - lambda self, conn, table_name, role=None: ( + lambda self, conn, table_name, schema_tag=None: ( "idx_person_id" if table_name == "person" else None ), ) diff --git a/tests/test_truncate_tables.py b/tests/test_truncate_tables.py index 0f97470..b871bfe 100644 --- a/tests/test_truncate_tables.py +++ b/tests/test_truncate_tables.py @@ -78,6 +78,7 @@ def fake_truncate_tables( restart_identities: bool = False, cascade: bool = False, dry_run: bool = False, + resolved: object = None, ) -> list[TruncateTableResult]: calls["engine"] = engine calls["scope"] = scope @@ -89,7 +90,7 @@ def fake_truncate_tables( TruncateTableResult( table_name="person", category=TableCategory.CLINICAL, - role=Role.PRIMARY, + schema_tag=Role.PRIMARY.value, row_count=10, status=Status.PLANNED, detail="table would be truncated", From ee61d59788e7466356291f0e05211ca95092d66a Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Wed, 23 Sep 2026 04:28:48 +0000 Subject: [PATCH 27/32] Follow-up from internal review --- omop_alchemy/maintenance/cli_fulltext.py | 11 +- omop_alchemy/maintenance/cli_indexes.py | 524 +++++++++--------- omop_alchemy/maintenance/cli_schema_tables.py | 12 +- omop_alchemy/maintenance/cli_tables.py | 23 +- omop_alchemy/maintenance/tables.py | 2 +- 5 files changed, 286 insertions(+), 286 deletions(-) diff --git a/omop_alchemy/maintenance/cli_fulltext.py b/omop_alchemy/maintenance/cli_fulltext.py index 6a358dd..f5a4993 100644 --- a/omop_alchemy/maintenance/cli_fulltext.py +++ b/omop_alchemy/maintenance/cli_fulltext.py @@ -2,6 +2,7 @@ from __future__ import annotations +from contextlib import ExitStack from dataclasses import dataclass from enum import StrEnum from typing import cast @@ -9,7 +10,7 @@ import typer import sqlalchemy as sa from sqlalchemy.engine import Engine -from oa_configurator import ResolvedCDMDatabase, Role, guard_schema_provenance_for, validate_schema_tag +from oa_configurator import ResolvedCDMDatabase, guard_schema_provenance_for, validate_schema_tag from ..backends import backend_support_note as _backend_support_note from ..backends import resolve_backend, require_backend_support @@ -89,11 +90,13 @@ def install_fulltext_columns( for cfg in targets: tag = _schema_tag_for_target(cfg.table_name) tables_by_schema_tag.setdefault(tag, []).append(_FULLTEXT_TARGET_TABLES[cfg.table_name]) - with engine.begin() as connection: + # One provenance guard per schema_tag (count only known at runtime); ExitStack defers every write until the block below succeeds. + with engine.begin() as connection, ExitStack() as guard_stack: for schema_tag, tables in tables_by_schema_tag.items(): # A same-named column/index could already exist under a drifted schema, attached to an unrelated table. - with guard_schema_provenance_for(connection, resolved, role=Role(schema_tag), tables=tables): - pass + guard_stack.enter_context( + guard_schema_provenance_for(connection, resolved, schema_tag=schema_tag, tables=tables) + ) for cfg in targets: backend.install_fulltext_on_table( connection, diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index fdd6720..f72116b 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +from contextlib import ExitStack from dataclasses import dataclass from typing import Any, Mapping, Sequence @@ -10,7 +11,7 @@ from sqlalchemy.exc import DBAPIError, IntegrityError import typer -from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance_for, schema_of, supports_schemas +from oa_configurator import ResolvedCDMDatabase, ensure_schema, guard_schema_provenance_for, schema_of, supports_schemas from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY @@ -696,294 +697,295 @@ def manage_indexes( metadata_indexes = _schema_metadata_indexes(selected_tables) clustering_supported = backend_supports(backend, "cluster_table") - if enable and not dry_run: - tables_by_schema_tag: dict[str, list[sa.Table]] = {} - for table in selected_tables: - tables_by_schema_tag.setdefault(table.schema_tag, []).append(table.table) - with engine.begin() as guard_connection: - for schema_tag, tables in tables_by_schema_tag.items(): - # A same-named index could already exist under a drifted schema, attached to an unrelated table. - with guard_schema_provenance_for( - guard_connection, resolved, role=Role(schema_tag), tables=tables - ): - pass - results: list[IndexManagementResult] = [] - for table in selected_tables: - db_schema = schema_of(engine, schema_tag=table.schema_tag) - if not inspector.has_table(table.table_name, schema=db_schema): - continue - - existing_indexes = inspector.get_indexes(table.table_name, schema=db_schema) - existing_index_names = {index["name"] for index in existing_indexes} + with ExitStack() as guard_stack: + if enable and not dry_run: + tables_by_schema_tag: dict[str, list[sa.Table]] = {} + for table in selected_tables: + tables_by_schema_tag.setdefault(table.schema_tag, []).append(table.table) + guard_connection = guard_stack.enter_context(engine.begin()) + # One provenance guard per schema_tag (count only known at runtime); ExitStack defers every write until the block below succeeds. + for schema_tag, tables in tables_by_schema_tag.items(): + # A same-named index could already exist under a drifted schema, attached to an unrelated table. + guard_stack.enter_context( + guard_schema_provenance_for(guard_connection, resolved, schema_tag=schema_tag, tables=tables) + ) - created_any = False - clustered_now = False - physical_index_names: dict[str, str] = {} + for table in selected_tables: + db_schema = schema_of(engine, schema_tag=table.schema_tag) + if not inspector.has_table(table.table_name, schema=db_schema): + continue - for metadata_index in sorted(table.table.indexes, key=lambda idx: idx.name or ""): - index_name = str(metadata_index.name) - column_names = tuple(column.name for column in metadata_index.columns) - unique = bool(metadata_index.unique) - exists = index_name in existing_index_names - should_apply = ( - not enable - ) or ( - enable and not exists - ) + existing_indexes = inspector.get_indexes(table.table_name, schema=db_schema) + existing_index_names = {index["name"] for index in existing_indexes} + + created_any = False + clustered_now = False + physical_index_names: dict[str, str] = {} + + for metadata_index in sorted(table.table.indexes, key=lambda idx: idx.name or ""): + index_name = str(metadata_index.name) + column_names = tuple(column.name for column in metadata_index.columns) + unique = bool(metadata_index.unique) + exists = index_name in existing_index_names + should_apply = ( + not enable + ) or ( + enable and not exists + ) - if not should_apply: - physical_index_names[index_name] = index_name - continue + if not should_apply: + physical_index_names[index_name] = index_name + continue - schema_index = metadata_indexes[(table.table_name, index_name)] - # Plain create/drop succeeding is the common case for both live and - # dry runs, so it's the default outcome; every branch below only - # overrides it for a foreign-index or already-in-place case. - outcome = _IndexOutcome( - status=dry_status(dry_run), - detail=dry_label( - dry_run, - planned="metadata-defined index would be dropped" if not enable else "metadata-defined index would be created", - applied="metadata-defined index dropped" if not enable else "metadata-defined index created", - ), - physical_name=index_name, - ) + schema_index = metadata_indexes[(table.table_name, index_name)] + # Plain create/drop succeeding is the common case for both live and + # dry runs, so it's the default outcome; every branch below only + # overrides it for a foreign-index or already-in-place case. + outcome = _IndexOutcome( + status=dry_status(dry_run), + detail=dry_label( + dry_run, + planned="metadata-defined index would be dropped" if not enable else "metadata-defined index would be created", + applied="metadata-defined index dropped" if not enable else "metadata-defined index created", + ), + physical_name=index_name, + ) - # Each index gets its own connection: a transaction when actually - # mutating (not dry_run, so WAL is committed and checkpointable - # before the next index build begins), a plain read-only - # connection when only previewing. - connection_factory = engine.begin if not dry_run else engine.connect - with connection_factory() as connection: - if not enable: - if not dry_run: - existed_before_drop = backend.index_exists( - connection, index_name, schema_tag=table.schema_tag - ) - else: - existed_before_drop = exists - if not existed_before_drop: - # Index under a different naming scheme than ours - equivalent_name = _find_equivalent_index(existing_indexes, column_names, unique) - if equivalent_name is not None: - if not dry_run: - captured = _record_captured_index( - connection, - table_name=table.table_name, db_schema=db_schema, - index_name=equivalent_name, - column_names=column_names, unique=unique, - ) - else: - pending_capture, _, _ = _peek_captured_index( - connection, - table_name=table.table_name, db_schema=db_schema, - column_names=column_names, unique=unique, - ) - captured = pending_capture is None - if captured: + # Each index gets its own connection: a transaction when actually + # mutating (not dry_run, so WAL is committed and checkpointable + # before the next index build begins), a plain read-only + # connection when only previewing. + connection_factory = engine.begin if not dry_run else engine.connect + with connection_factory() as connection: + if not enable: + if not dry_run: + existed_before_drop = backend.index_exists( + connection, index_name, schema_tag=table.schema_tag + ) + else: + existed_before_drop = exists + if not existed_before_drop: + # Index under a different naming scheme than ours + equivalent_name = _find_equivalent_index(existing_indexes, column_names, unique) + if equivalent_name is not None: if not dry_run: - backend.drop_index_if_exists( - connection, equivalent_name, schema_tag=table.schema_tag + captured = _record_captured_index( + connection, + table_name=table.table_name, db_schema=db_schema, + index_name=equivalent_name, + column_names=column_names, unique=unique, ) - outcome = _IndexOutcome( - status=dry_status(dry_run, Status.CAPTURED), - detail=dry_label( - dry_run, - planned=f"foreign index '{equivalent_name}' would be captured and dropped for bulk load", - applied=f"foreign index '{equivalent_name}' captured and dropped for bulk load", - ), - physical_name=equivalent_name, - ) - else: - # A different foreign index for this table/column-set is - # already captured and awaiting restore. Leave this one - # in place rather than dropping something we can no - # longer track. - outcome = _IndexOutcome( - status=Status.WARNING, - detail=dry_label( - dry_run, - planned=( - f"foreign index '{equivalent_name}' would be left in place: a different " - "foreign index for this table/column-set is already captured and " - "awaiting restore" + else: + pending_capture, _, _ = _peek_captured_index( + connection, + table_name=table.table_name, db_schema=db_schema, + column_names=column_names, unique=unique, + ) + captured = pending_capture is None + if captured: + if not dry_run: + backend.drop_index_if_exists( + connection, equivalent_name, schema_tag=table.schema_tag + ) + outcome = _IndexOutcome( + status=dry_status(dry_run, Status.CAPTURED), + detail=dry_label( + dry_run, + planned=f"foreign index '{equivalent_name}' would be captured and dropped for bulk load", + applied=f"foreign index '{equivalent_name}' captured and dropped for bulk load", ), - applied=( - f"foreign index '{equivalent_name}' left in place: a different " - "foreign index for this table/column-set is already captured and " - "awaiting restore" + physical_name=equivalent_name, + ) + else: + # A different foreign index for this table/column-set is + # already captured and awaiting restore. Leave this one + # in place rather than dropping something we can no + # longer track. + outcome = _IndexOutcome( + status=Status.WARNING, + detail=dry_label( + dry_run, + planned=( + f"foreign index '{equivalent_name}' would be left in place: a different " + "foreign index for this table/column-set is already captured and " + "awaiting restore" + ), + applied=( + f"foreign index '{equivalent_name}' left in place: a different " + "foreign index for this table/column-set is already captured and " + "awaiting restore" + ), ), - ), - physical_name=equivalent_name, - ) - else: - conflict = _find_shape_conflict(existing_indexes, column_names, unique) - if conflict is not None: - conflict_name = str(conflict["name"]) - outcome = _IndexOutcome( - status=Status.WARNING, - detail=dry_label( - dry_run, - planned=f"foreign index '{conflict_name}' {_describe_shape_conflict(conflict)}; would be left in place", - applied=f"foreign index '{conflict_name}' {_describe_shape_conflict(conflict)}; left in place", - ), - physical_name=conflict_name, - ) + physical_name=equivalent_name, + ) else: - outcome = _IndexOutcome( - status=Status.SKIPPED, - detail="metadata-defined index already absent (skipped)", - physical_name=index_name, - ) - elif not dry_run: - backend.drop_index_if_exists(connection, index_name, schema_tag=table.schema_tag) - # outcome stays default: applied / "metadata-defined index dropped" - # dry-run, existed_before_drop True: outcome stays default ("would be dropped") - else: - if not dry_run: - restored_name = _restore_captured_index( - connection, - table_name=table.table_name, db_schema=db_schema, - column_names=column_names, unique=unique, - ) + conflict = _find_shape_conflict(existing_indexes, column_names, unique) + if conflict is not None: + conflict_name = str(conflict["name"]) + outcome = _IndexOutcome( + status=Status.WARNING, + detail=dry_label( + dry_run, + planned=f"foreign index '{conflict_name}' {_describe_shape_conflict(conflict)}; would be left in place", + applied=f"foreign index '{conflict_name}' {_describe_shape_conflict(conflict)}; left in place", + ), + physical_name=conflict_name, + ) + else: + outcome = _IndexOutcome( + status=Status.SKIPPED, + detail="metadata-defined index already absent (skipped)", + physical_name=index_name, + ) + elif not dry_run: + backend.drop_index_if_exists(connection, index_name, schema_tag=table.schema_tag) + # outcome stays default: applied / "metadata-defined index dropped" + # dry-run, existed_before_drop True: outcome stays default ("would be dropped") else: - restored_name, _, _ = _peek_captured_index( - connection, - table_name=table.table_name, db_schema=db_schema, - column_names=column_names, unique=unique, - ) - if restored_name is not None: if not dry_run: - created_any = True - outcome = _IndexOutcome( - status=dry_status(dry_run, Status.RESTORED), - detail=dry_label( - dry_run, - planned=f"foreign index '{restored_name}' would be restored from bulk-load capture", - applied=f"foreign index '{restored_name}' restored from bulk-load capture", - ), - physical_name=restored_name, - ) - else: - equivalent_name = _find_equivalent_index(existing_indexes, column_names, unique) - if equivalent_name is not None: + restored_name = _restore_captured_index( + connection, + table_name=table.table_name, db_schema=db_schema, + column_names=column_names, unique=unique, + ) + else: + restored_name, _, _ = _peek_captured_index( + connection, + table_name=table.table_name, db_schema=db_schema, + column_names=column_names, unique=unique, + ) + if restored_name is not None: + if not dry_run: + created_any = True outcome = _IndexOutcome( - status=Status.SKIPPED, + status=dry_status(dry_run, Status.RESTORED), detail=dry_label( dry_run, - planned=f"equivalent foreign index '{equivalent_name}' already provides this coverage (would skip creation)", - applied=f"equivalent foreign index '{equivalent_name}' already provides this coverage (skipped)", + planned=f"foreign index '{restored_name}' would be restored from bulk-load capture", + applied=f"foreign index '{restored_name}' restored from bulk-load capture", ), - physical_name=equivalent_name, + physical_name=restored_name, ) - elif not dry_run: - savepoint = connection.begin_nested() - try: - schema_index.create(bind=connection, checkfirst=True) - except DBAPIError as exc: - savepoint.rollback() - if "already exists" not in str(exc.orig).lower(): - raise + else: + equivalent_name = _find_equivalent_index(existing_indexes, column_names, unique) + if equivalent_name is not None: outcome = _IndexOutcome( status=Status.SKIPPED, - detail="metadata-defined index already exists (skipped)", - physical_name=index_name, + detail=dry_label( + dry_run, + planned=f"equivalent foreign index '{equivalent_name}' already provides this coverage (would skip creation)", + applied=f"equivalent foreign index '{equivalent_name}' already provides this coverage (skipped)", + ), + physical_name=equivalent_name, ) - else: - savepoint.commit() - created_any = True - # outcome stays default: applied / "metadata-defined index created" - # dry-run, no restore, no equivalent: outcome stays default ("would be created") - - physical_name = outcome.physical_name - physical_index_names[index_name] = physical_name - results.append( - IndexManagementResult( - operation="index", - table_name=table.table_name, - category=table.category, - schema_tag=table.schema_tag, - index_name=physical_name, - column_names=column_names, - unique=unique, - clustered=metadata_index.info.get(OMOP_CLUSTER_INDEX_INFO_KEY) is True, - enable=enable, - status=outcome.status, - detail=outcome.detail, + elif not dry_run: + savepoint = connection.begin_nested() + try: + schema_index.create(bind=connection, checkfirst=True) + except DBAPIError as exc: + savepoint.rollback() + if "already exists" not in str(exc.orig).lower(): + raise + outcome = _IndexOutcome( + status=Status.SKIPPED, + detail="metadata-defined index already exists (skipped)", + physical_name=index_name, + ) + else: + savepoint.commit() + created_any = True + # outcome stays default: applied / "metadata-defined index created" + # dry-run, no restore, no equivalent: outcome stays default ("would be created") + + physical_name = outcome.physical_name + physical_index_names[index_name] = physical_name + results.append( + IndexManagementResult( + operation="index", + table_name=table.table_name, + category=table.category, + schema_tag=table.schema_tag, + index_name=physical_name, + column_names=column_names, + unique=unique, + clustered=metadata_index.info.get(OMOP_CLUSTER_INDEX_INFO_KEY) is True, + enable=enable, + status=outcome.status, + detail=outcome.detail, + ) ) - ) - # Clustering for perfomance is a separate operation from index creation - if enable: - cluster_index_name = _cluster_target_name(table) - if cluster_index_name is not None: - cluster_columns = _cluster_column_names(table, cluster_index_name) - if cluster_index_name in physical_index_names: - # Resolved authoritatively from what actually happened in this - # run's per-index loop (own name, captured, restored, or a - # skip-equivalent) -- more precise than re-deriving from the - # now-stale existing_indexes snapshot, since e.g. a - # just-restored index wouldn't appear in it. - physical_cluster_name = physical_index_names[cluster_index_name] - else: - # Primary-key-based cluster target: never entered the per-index - # loop, so resolve it the same way the standalone `indexes - # cluster` command does. - physical_cluster_name = _resolve_physical_cluster_name( - existing_indexes, - cluster_index_name, - cluster_columns, - ) - if not clustering_supported or not cluster: - results.append( - IndexManagementResult( - operation="cluster", - table_name=table.table_name, - category=table.category, - schema_tag=table.schema_tag, - index_name=physical_cluster_name, - column_names=cluster_columns, - unique=False, - clustered=True, - enable=enable, - status=Status.SKIPPED, - detail=( - f"cluster metadata present but unsupported on {backend.name}" - if not clustering_supported - else "clustering skipped (run 'indexes cluster' to apply)" - ), + # Clustering for perfomance is a separate operation from index creation + if enable: + cluster_index_name = _cluster_target_name(table) + if cluster_index_name is not None: + cluster_columns = _cluster_column_names(table, cluster_index_name) + if cluster_index_name in physical_index_names: + # Resolved authoritatively from what actually happened in this + # run's per-index loop (own name, captured, restored, or a + # skip-equivalent) -- more precise than re-deriving from the + # now-stale existing_indexes snapshot, since e.g. a + # just-restored index wouldn't appear in it. + physical_cluster_name = physical_index_names[cluster_index_name] + else: + # Primary-key-based cluster target: never entered the per-index + # loop, so resolve it the same way the standalone `indexes + # cluster` command does. + physical_cluster_name = _resolve_physical_cluster_name( + existing_indexes, + cluster_index_name, + cluster_columns, ) - ) - else: - if not dry_run: - with engine.begin() as connection: - backend.cluster_table( - connection, table.table_name, physical_cluster_name, schema_tag=table.schema_tag + if not clustering_supported or not cluster: + results.append( + IndexManagementResult( + operation="cluster", + table_name=table.table_name, + category=table.category, + schema_tag=table.schema_tag, + index_name=physical_cluster_name, + column_names=cluster_columns, + unique=False, + clustered=True, + enable=enable, + status=Status.SKIPPED, + detail=( + f"cluster metadata present but unsupported on {backend.name}" + if not clustering_supported + else "clustering skipped (run 'indexes cluster' to apply)" + ), + ) + ) + else: + if not dry_run: + with engine.begin() as connection: + backend.cluster_table( + connection, table.table_name, physical_cluster_name, schema_tag=table.schema_tag + ) + clustered_now = True + + results.append( + IndexManagementResult( + operation="cluster", + table_name=table.table_name, + category=table.category, + schema_tag=table.schema_tag, + index_name=physical_cluster_name, + column_names=cluster_columns, + unique=False, + clustered=True, + enable=enable, + status=dry_status(dry_run), + detail=dry_label(dry_run, "table would be clustered using ORM-defined metadata", "table clustered using ORM-defined metadata"), ) - clustered_now = True - - results.append( - IndexManagementResult( - operation="cluster", - table_name=table.table_name, - category=table.category, - schema_tag=table.schema_tag, - index_name=physical_cluster_name, - column_names=cluster_columns, - unique=False, - clustered=True, - enable=enable, - status=dry_status(dry_run), - detail=dry_label(dry_run, "table would be clustered using ORM-defined metadata", "table clustered using ORM-defined metadata"), ) - ) - if not dry_run and (created_any or clustered_now): - with engine.connect() as connection: - backend.analyze_table(connection, table.table_name, schema_tag=table.schema_tag) - connection.commit() + if not dry_run and (created_any or clustered_now): + with engine.connect() as connection: + backend.analyze_table(connection, table.table_name, schema_tag=table.schema_tag) + connection.commit() return results diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index 521db23..f653b4f 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -120,9 +120,9 @@ def create_missing_tables( # One call: create_all's dependency sort and FK-deferral must see every table together. with ( engine.begin() as connection, - guard_schema_provenance_for(connection, resolved, role=Role.PRIMARY, tables=tables_by_role[Role.PRIMARY]), - guard_schema_provenance_for(connection, resolved, role=Role.RESULTS, tables=tables_by_role[Role.RESULTS]), - guard_schema_provenance_for(connection, resolved, role=Role.VOCAB, tables=tables_by_role[Role.VOCAB]), + guard_schema_provenance_for(connection, resolved, schema_tag=Role.PRIMARY, tables=tables_by_role[Role.PRIMARY]), + guard_schema_provenance_for(connection, resolved, schema_tag=Role.RESULTS, tables=tables_by_role[Role.RESULTS]), + guard_schema_provenance_for(connection, resolved, schema_tag=Role.VOCAB, tables=tables_by_role[Role.VOCAB]), ): Base.metadata.create_all( bind=connection, tables=all_tables, checkfirst=True @@ -135,8 +135,8 @@ def create_missing_tables( if other_tables: with ( engine.begin() as connection, - guard_schema_provenance_for(connection, resolved, role=Role.PRIMARY, tables=tables_by_role[Role.PRIMARY]), - guard_schema_provenance_for(connection, resolved, role=Role.RESULTS, tables=tables_by_role[Role.RESULTS]), + guard_schema_provenance_for(connection, resolved, schema_tag=Role.PRIMARY, tables=tables_by_role[Role.PRIMARY]), + guard_schema_provenance_for(connection, resolved, schema_tag=Role.RESULTS, tables=tables_by_role[Role.RESULTS]), ): Base.metadata.create_all( bind=connection, tables=other_tables, checkfirst=True @@ -144,7 +144,7 @@ def create_missing_tables( if vocab_tables: with ( vocab_engine.begin() as vocab_connection, - guard_schema_provenance_for(vocab_connection, resolved, role=Role.VOCAB, tables=vocab_tables), + guard_schema_provenance_for(vocab_connection, resolved, schema_tag=Role.VOCAB, tables=vocab_tables), ): Base.metadata.create_all( bind=vocab_connection, tables=vocab_tables, checkfirst=True diff --git a/omop_alchemy/maintenance/cli_tables.py b/omop_alchemy/maintenance/cli_tables.py index 55dcc69..e4fb12f 100644 --- a/omop_alchemy/maintenance/cli_tables.py +++ b/omop_alchemy/maintenance/cli_tables.py @@ -237,27 +237,22 @@ def truncate_tables( raise RuntimeError(_format_blocking_reference_error(blockers)) if existing_tables and not dry_run: - # One TRUNCATE per schema tag: truncate_table_batch qualifies every - # name in its list with a single tag, so a selection spanning more - # than one tag (category and schema_tag are independent axes -- - # see MaintenanceTable.schema_tag) is split into one batch per tag - # rather than misqualifying some of the names. + # One TRUNCATE batch per schema_tag, since truncate_table_batch qualifies its whole list with a single tag. for schema_tag, table_names_for_tag in existing_table_names_by_schema_tag.items(): # A same-named table could already exist under a drifted schema, so truncate could hit unrelated data. with guard_schema_provenance_for( connection, resolved, - role=Role(schema_tag), + schema_tag=schema_tag, tables=[tables_by_name[name].table for name in table_names_for_tag], ): - pass - backend.truncate_table_batch( - connection, - table_names_for_tag, - restart_identities=restart_identities, - cascade=cascade, - schema_tag=schema_tag, - ) + backend.truncate_table_batch( + connection, + table_names_for_tag, + restart_identities=restart_identities, + cascade=cascade, + schema_tag=schema_tag, + ) return results diff --git a/omop_alchemy/maintenance/tables.py b/omop_alchemy/maintenance/tables.py index ce13192..2d115dd 100644 --- a/omop_alchemy/maintenance/tables.py +++ b/omop_alchemy/maintenance/tables.py @@ -20,7 +20,7 @@ class TableCategory(StrEnum): Notes ----- This logical grouping is independent of the physical schema in - which the table's data is stored (``oa_configurator.Role``). + which the table's data is stored (its own ``schema_tag``). Parameters ---------- From eb5baa17f7c7c8fdfe2093af9cd2f4d33adbc55d Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Wed, 23 Sep 2026 04:47:41 +0000 Subject: [PATCH 28/32] Disambiguate physical schema from schema tag --- omop_alchemy/backends/base.py | 2 +- omop_alchemy/backends/postgres.py | 38 +++++++++---------- omop_alchemy/maintenance/cli_foreign_keys.py | 6 +-- omop_alchemy/maintenance/cli_indexes.py | 8 ++-- .../maintenance/cli_schema_reconcile.py | 10 ++--- .../maintenance/cli_schema_summary.py | 6 +-- omop_alchemy/maintenance/cli_schema_tables.py | 6 +-- omop_alchemy/maintenance/cli_tables.py | 14 +++---- omop_alchemy/maintenance/tables.py | 8 ++-- ...st_backends_non_default_schema_postgres.py | 4 +- tests/test_schema_reconcile.py | 16 ++++---- 11 files changed, 59 insertions(+), 59 deletions(-) diff --git a/omop_alchemy/backends/base.py b/omop_alchemy/backends/base.py index 1b31295..49db32c 100644 --- a/omop_alchemy/backends/base.py +++ b/omop_alchemy/backends/base.py @@ -159,7 +159,7 @@ def approximate_row_counts( """Cheap, catalog-based row-count estimate per table in schema. Unlike most Backend methods, schema is explicit rather than read via - schema_of(conn): a caller previewing an orphan schema is inspecting a + physical_schema_of(conn): a caller previewing an orphan schema is inspecting a schema other than the connection's own configured one. """ raise FeatureNotSupportedError("Approximate row counts", self) diff --git a/omop_alchemy/backends/postgres.py b/omop_alchemy/backends/postgres.py index 14c2791..4ee12b6 100644 --- a/omop_alchemy/backends/postgres.py +++ b/omop_alchemy/backends/postgres.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from oa_configurator import Dialect, Role, qualified, schema_of +from oa_configurator import Dialect, Role, qualified, physical_schema_of from sqlalchemy.dialects.postgresql import REGCONFIG, TSVECTOR from sqlalchemy.sql import func @@ -38,7 +38,7 @@ def toggle_fk_triggers( ) -> None: action = "ENABLE" if enable else "DISABLE" conn.exec_driver_sql( - f"ALTER TABLE {qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag))} {action} TRIGGER ALL" + f"ALTER TABLE {qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))} {action} TRIGGER ALL" ) def get_fk_trigger_counts( @@ -63,7 +63,7 @@ def get_fk_trigger_counts( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": schema_of(conn, schema_tag=schema_tag)}, + {"table_name": table_name, "db_schema": physical_schema_of(conn, schema_tag=schema_tag)}, ).one() return int(disabled_count or 0), int(enabled_count or 0) @@ -78,8 +78,8 @@ def count_fk_violations( source_schema_tag: str = Role.PRIMARY.value, referred_schema_tag: str = Role.PRIMARY.value, ) -> int: - source = qualified(conn, source_table, physical_schema=schema_of(conn, schema_tag=source_schema_tag)) - referred = qualified(conn, referred_table, physical_schema=schema_of(conn, schema_tag=referred_schema_tag)) + source = qualified(conn, source_table, physical_schema=physical_schema_of(conn, schema_tag=source_schema_tag)) + referred = qualified(conn, referred_table, physical_schema=physical_schema_of(conn, schema_tag=referred_schema_tag)) non_null_predicate = " AND ".join( f"src.{col} IS NOT NULL" for col in constrained_cols ) @@ -113,7 +113,7 @@ def cluster_table( schema_tag: str = Role.PRIMARY.value, ) -> None: conn.exec_driver_sql( - f"CLUSTER {qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag))} USING {index_name}" + f"CLUSTER {qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))} USING {index_name}" ) def get_clustered_index_name( @@ -136,7 +136,7 @@ def get_clustered_index_name( AND (CAST(:db_schema AS TEXT) IS NULL OR n.nspname = :db_schema) """ ), - {"table_name": table_name, "db_schema": schema_of(conn, schema_tag=schema_tag)}, + {"table_name": table_name, "db_schema": physical_schema_of(conn, schema_tag=schema_tag)}, ).scalar_one_or_none() return str(result) if result is not None else None @@ -190,7 +190,7 @@ def analyze_table( schema_tag: str = Role.PRIMARY.value, ) -> None: operation = "VACUUM ANALYZE" if vacuum else "ANALYZE" - conn.exec_driver_sql(f"{operation} {qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag))}") + conn.exec_driver_sql(f"{operation} {qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))}") def index_exists( self, @@ -199,7 +199,7 @@ def index_exists( *, schema_tag: str = Role.PRIMARY.value, ) -> bool: - qualified_index_name = qualified(conn, index_name, physical_schema=schema_of(conn, schema_tag=schema_tag)) + qualified_index_name = qualified(conn, index_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag)) return bool( conn.scalar( sa.select( @@ -211,7 +211,7 @@ def index_exists( def drop_index_if_exists( self, conn: sa.Connection, index_name: str, *, schema_tag: str = Role.PRIMARY.value ) -> None: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, physical_schema=schema_of(conn, schema_tag=schema_tag))}") + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))}") def truncate_table_batch( self, @@ -223,7 +223,7 @@ def truncate_table_batch( schema_tag: str = Role.PRIMARY.value, ) -> None: sql = "TRUNCATE TABLE " + ", ".join( - qualified(conn, name, physical_schema=schema_of(conn, schema_tag=schema_tag)) for name in table_names + qualified(conn, name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag)) for name in table_names ) if restart_identities: sql += " RESTART IDENTITY" @@ -241,7 +241,7 @@ def find_sequence_name( *, schema_tag: str = Role.PRIMARY.value, ) -> str | None: - fully_qualified = qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag)) + fully_qualified = qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag)) return conn.execute( sa.text("SELECT pg_get_serial_sequence(:table_name, :column_name)"), {"table_name": fully_qualified, "column_name": column_name}, @@ -331,7 +331,7 @@ def install_fulltext_on_table( fastupdate: bool, schema_tag: str = Role.PRIMARY.value, ) -> None: - qualified_table = qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag)) + qualified_table = qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag)) conn.exec_driver_sql( f"ALTER TABLE {qualified_table} ADD COLUMN IF NOT EXISTS {vector_column_name} tsvector" ) @@ -340,7 +340,7 @@ def install_fulltext_on_table( table_name, sa.MetaData(), sa.Column(vector_column_name, TSVECTOR), - schema=schema_of(conn, schema_tag=schema_tag), + schema=physical_schema_of(conn, schema_tag=schema_tag), ) index = sa.Index( index_name, @@ -364,7 +364,7 @@ def populate_fulltext_on_table( table_name, sa.column(vector_column_name), sa.column(source_column_name), - schema=schema_of(conn, schema_tag=schema_tag), + schema=physical_schema_of(conn, schema_tag=schema_tag), ) source_column = lightweight_table.c[source_column_name] stmt = lightweight_table.update().values( @@ -391,9 +391,9 @@ def drop_fulltext_on_table( schema_tag: str = Role.PRIMARY.value, ) -> None: if drop_indexes: - conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, physical_schema=schema_of(conn, schema_tag=schema_tag))}") + conn.exec_driver_sql(f"DROP INDEX IF EXISTS {qualified(conn, index_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))}") conn.exec_driver_sql( - f"ALTER TABLE {qualified(conn, table_name, physical_schema=schema_of(conn, schema_tag=schema_tag))}" + f"ALTER TABLE {qualified(conn, table_name, physical_schema=physical_schema_of(conn, schema_tag=schema_tag))}" f" DROP COLUMN IF EXISTS {vector_column_name}" ) @@ -424,7 +424,7 @@ def prepare_backup( "--no-owner", "--no-privileges", ] - db_schema = schema_of(engine, schema_tag=schema_tag) + db_schema = physical_schema_of(engine, schema_tag=schema_tag) if db_schema: command.extend(["--schema", db_schema]) env = os.environ.copy() @@ -447,7 +447,7 @@ def prepare_restore( "Database restore requires a database name in the configured engine URL." ) connection_uri = _libpq_connection_uri(url) - db_schema = schema_of(engine, schema_tag=schema_tag) + db_schema = physical_schema_of(engine, schema_tag=schema_tag) if backup_format == "custom": tool_path = _pg_restore_path() diff --git a/omop_alchemy/maintenance/cli_foreign_keys.py b/omop_alchemy/maintenance/cli_foreign_keys.py index a977326..b42e333 100644 --- a/omop_alchemy/maintenance/cli_foreign_keys.py +++ b/omop_alchemy/maintenance/cli_foreign_keys.py @@ -7,7 +7,7 @@ import sqlalchemy as sa import typer -from oa_configurator import schema_of +from oa_configurator import physical_schema_of from ..backends import Backend, resolve_backend, require_backend_support, backend_support_note from ._cli_utils import Status, dry_label, dry_status, omop_command from .tables import ( @@ -106,7 +106,7 @@ def _collect_fk_info( for table_name in selected_names: foreign_keys = inspector.get_foreign_keys( - table_name, schema=schema_of(engine, schema_tag=tables_by_name[table_name].schema_tag) + table_name, schema=physical_schema_of(engine, schema_tag=tables_by_name[table_name].schema_tag) ) relevant_foreign_keys = [ foreign_key @@ -166,7 +166,7 @@ def _collect_strict_validation_failures( for table_name in sorted(selected_names): source_schema_tag = tables_by_name[table_name].schema_tag for foreign_key in inspector.get_foreign_keys( - table_name, schema=schema_of(connection, schema_tag=source_schema_tag) + table_name, schema=physical_schema_of(connection, schema_tag=source_schema_tag) ): referred_table = foreign_key.get("referred_table") constrained_columns = foreign_key.get("constrained_columns") or [] diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index f72116b..b4469af 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -11,7 +11,7 @@ from sqlalchemy.exc import DBAPIError, IntegrityError import typer -from oa_configurator import ResolvedCDMDatabase, ensure_schema, guard_schema_provenance_for, schema_of, supports_schemas +from oa_configurator import ResolvedCDMDatabase, ensure_schema, guard_schema_provenance_for, physical_schema_of, supports_schemas from omop_alchemy.cdm.base.indexing import OMOP_CLUSTER_INDEX_INFO_KEY @@ -624,7 +624,7 @@ def collect_index_targets( targets: list[IndexTarget] = [] for table in selected_tables: - table_schema = schema_of(engine, schema_tag=table.schema_tag) + table_schema = physical_schema_of(engine, schema_tag=table.schema_tag) if not inspector.has_table(table.table_name, schema=table_schema): continue @@ -713,7 +713,7 @@ def manage_indexes( ) for table in selected_tables: - db_schema = schema_of(engine, schema_tag=table.schema_tag) + db_schema = physical_schema_of(engine, schema_tag=table.schema_tag) if not inspector.has_table(table.table_name, schema=db_schema): continue @@ -1089,7 +1089,7 @@ def cluster_tables_command( results: list[IndexManagementResult] = [] for table in selected_tables: - table_schema = schema_of(engine, schema_tag=table.schema_tag) + table_schema = physical_schema_of(engine, schema_tag=table.schema_tag) if not inspector.has_table(table.table_name, schema=table_schema): continue diff --git a/omop_alchemy/maintenance/cli_schema_reconcile.py b/omop_alchemy/maintenance/cli_schema_reconcile.py index e1d4107..d34ef47 100644 --- a/omop_alchemy/maintenance/cli_schema_reconcile.py +++ b/omop_alchemy/maintenance/cli_schema_reconcile.py @@ -8,7 +8,7 @@ from oa_configurator import ( ResolvedDatabase, find_table_in_other_schemas, - schema_of, + physical_schema_of, supports_schemas, validate_schema_tag, ) @@ -79,13 +79,13 @@ def _effective_schema( schema_tag: str | None, db_schema: str | None, ) -> str | None: - """schema_of(engine, schema_tag=schema_tag) when resolved is given, else db_schema. + """physical_schema_of(engine, schema_tag=schema_tag) when resolved is given, else db_schema. - Uses schema_of against engine to accommodate bare schema_tags. + Uses physical_schema_of against engine to accommodate bare schema_tags. """ if resolved is None: return db_schema - return schema_of(engine, schema_tag=schema_tag) + return physical_schema_of(engine, schema_tag=schema_tag) def _schema_qualified_tables( @@ -258,7 +258,7 @@ def reconcile_schema( exists = inspector.has_table(maintenance_table.table_name, schema=table_schema) if not exists: relocated_to = find_table_in_other_schemas( - engine, maintenance_table.table_name, expected_schema=table_schema + engine, maintenance_table.table_name, physical_schema=table_schema ) if relocated_to: detail = ( diff --git a/omop_alchemy/maintenance/cli_schema_summary.py b/omop_alchemy/maintenance/cli_schema_summary.py index 4dc5ad0..89fe89c 100644 --- a/omop_alchemy/maintenance/cli_schema_summary.py +++ b/omop_alchemy/maintenance/cli_schema_summary.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from oa_configurator import qualified, schema_of +from oa_configurator import qualified, physical_schema_of from .tables import TableCategory, select_omop_tables @@ -36,7 +36,7 @@ def collect_data_summary( results: list[TableSummaryResult] = [] with engine.connect() as connection: for table in tables: - exists = inspector.has_table(table.table_name, schema=schema_of(engine, schema_tag=table.schema_tag)) + exists = inspector.has_table(table.table_name, schema=physical_schema_of(engine, schema_tag=table.schema_tag)) if not exists and existing_only: continue @@ -45,7 +45,7 @@ def collect_data_summary( row_count = int( connection.execute( sa.text( - f"SELECT COUNT(*) FROM {qualified(connection, table.table_name, physical_schema=schema_of(connection, schema_tag=table.schema_tag))}" + f"SELECT COUNT(*) FROM {qualified(connection, table.table_name, physical_schema=physical_schema_of(connection, schema_tag=table.schema_tag))}" ) ).scalar_one() ) diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index f653b4f..ccf1d1e 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -6,7 +6,7 @@ import sqlalchemy as sa -from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance_for, schema_of +from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance_for, physical_schema_of from orm_loader.helpers import Base from ._cli_utils import Status, dry_label, dry_status from .tables import ( @@ -76,7 +76,7 @@ def create_missing_tables( """ vocab_engine = vocab_engine if vocab_engine is not None else engine if not dry_run: - ensure_schema(engine, schema_of(engine, schema_tag=Role.PRIMARY)) + ensure_schema(engine, physical_schema_of(engine, schema_tag=Role.PRIMARY)) # create_all() would fail for non-existing vocab/results schemas on a fresh database if resolved is not None: ensure_schema(engine, resolved.schema_for_role(Role.RESULTS)) @@ -89,7 +89,7 @@ def create_missing_tables( # Checking only primary schema would hide existing vocab/results tables, wrongly blocking dependents. existing_table_names: set[str] = set() for role in Role: - existing_table_names |= set(inspector.get_table_names(schema=schema_of(engine, schema_tag=role))) + existing_table_names |= set(inspector.get_table_names(schema=physical_schema_of(engine, schema_tag=role))) missing_table_names = {table.table_name for table in missing_tables} blocked_dependencies: dict[str, tuple[str, ...]] = {} diff --git a/omop_alchemy/maintenance/cli_tables.py b/omop_alchemy/maintenance/cli_tables.py index e4fb12f..959b014 100644 --- a/omop_alchemy/maintenance/cli_tables.py +++ b/omop_alchemy/maintenance/cli_tables.py @@ -7,7 +7,7 @@ import sqlalchemy as sa import typer -from oa_configurator import ResolvedCDMDatabase, Role, autocommit_connection, guard_schema_provenance_for, qualified, schema_of +from oa_configurator import ResolvedCDMDatabase, Role, autocommit_connection, guard_schema_provenance_for, qualified, physical_schema_of from ..backends import resolve_backend, require_backend_support, backend_support_note from ._cli_utils import Status, dry_label, dry_status, omop_command, resolve_selection from .tables import ( @@ -70,7 +70,7 @@ def analyze_tables( with connection_factory as connection: for maintenance_table in selected_tables: - table_schema = schema_of(engine, schema_tag=maintenance_table.schema_tag) + table_schema = physical_schema_of(engine, schema_tag=maintenance_table.schema_tag) if not inspector.has_table(maintenance_table.table_name, schema=table_schema): results.append( AnalyzeTableResult( @@ -134,7 +134,7 @@ def _blocking_foreign_key_references( blockers: dict[str, set[str]] = {} for role in Role: - role_schema = schema_of(engine, schema_tag=role) + role_schema = physical_schema_of(engine, schema_tag=role) for table_name in inspector.get_table_names(schema=role_schema): if table_name in selected_table_names: continue @@ -193,7 +193,7 @@ def truncate_tables( with engine.begin() as connection: for maintenance_table in selected_tables: if not inspector.has_table( - maintenance_table.table_name, schema=schema_of(engine, schema_tag=maintenance_table.schema_tag) + maintenance_table.table_name, schema=physical_schema_of(engine, schema_tag=maintenance_table.schema_tag) ): results.append( TruncateTableResult( @@ -209,7 +209,7 @@ def truncate_tables( row_count = int( connection.exec_driver_sql( - f"SELECT COUNT(*) FROM {qualified(connection, maintenance_table.table_name, physical_schema=schema_of(connection, schema_tag=maintenance_table.schema_tag))}" + f"SELECT COUNT(*) FROM {qualified(connection, maintenance_table.table_name, physical_schema=physical_schema_of(connection, schema_tag=maintenance_table.schema_tag))}" ).scalar_one() ) existing_tables.append(maintenance_table.table_name) @@ -324,7 +324,7 @@ def reset_model_sequences( with engine.begin() as connection: for target in targets: - if not inspector.has_table(target.table_name, schema=schema_of(engine, schema_tag=target.schema_tag)): + if not inspector.has_table(target.table_name, schema=physical_schema_of(engine, schema_tag=target.schema_tag)): continue sequence_name = backend.find_sequence_name( @@ -347,7 +347,7 @@ def reset_model_sequences( continue fully_qualified = qualified( - connection, target.table_name, physical_schema=schema_of(connection, schema_tag=target.schema_tag) + connection, target.table_name, physical_schema=physical_schema_of(connection, schema_tag=target.schema_tag) ) current_max = connection.execute( sa.text( diff --git a/omop_alchemy/maintenance/tables.py b/omop_alchemy/maintenance/tables.py index 2d115dd..3f815e2 100644 --- a/omop_alchemy/maintenance/tables.py +++ b/omop_alchemy/maintenance/tables.py @@ -5,7 +5,7 @@ from typing import Iterable import sqlalchemy as sa -from oa_configurator import schema_of, validate_schema_tag +from oa_configurator import physical_schema_of, validate_schema_tag class TableCategory(StrEnum): @@ -262,7 +262,7 @@ def existing_maintenance_tables( bindable : sqlalchemy.Engine or sqlalchemy.Connection Used both to inspect the database and, via its schema_translate_map, to resolve each table's own schema tag to a physical schema - (``schema_of(bindable, schema_tag=table.schema_tag)``) -- a blanket + (``physical_schema_of(bindable, schema_tag=table.schema_tag)``) -- a blanket schema passed in once would silently misclassify every vocab/results table checked against a database with a genuine primary/vocab/ results split. @@ -274,7 +274,7 @@ def existing_maintenance_tables( vocabulary_included=vocabulary_included, require_single_integer_primary_key=require_single_integer_primary_key, ) - if inspector.has_table(table.table_name, schema=schema_of(bindable, schema_tag=table.schema_tag)) + if inspector.has_table(table.table_name, schema=physical_schema_of(bindable, schema_tag=table.schema_tag)) ] @@ -292,5 +292,5 @@ def missing_maintenance_tables( return [ table for table in select_omop_tables(vocabulary_included=vocabulary_included) - if not inspector.has_table(table.table_name, schema=schema_of(bindable, schema_tag=table.schema_tag)) + if not inspector.has_table(table.table_name, schema=physical_schema_of(bindable, schema_tag=table.schema_tag)) ] diff --git a/tests/test_backends_non_default_schema_postgres.py b/tests/test_backends_non_default_schema_postgres.py index 299e10c..5655303 100644 --- a/tests/test_backends_non_default_schema_postgres.py +++ b/tests/test_backends_non_default_schema_postgres.py @@ -1,10 +1,10 @@ """Non-default-schema Postgres coverage for the backends/ signature refactor (Phase 3.2). Every other maintenance-CLI test runs against the default schema, where -``schema_of(conn)`` returning ``None`` and the old ``db_schema=None`` +``physical_schema_of(conn)`` returning ``None`` and the old ``db_schema=None`` parameter are indistinguishable. The refactor that dropped explicit ``db_schema`` threading through ``backends/`` (deriving it internally via -``schema_of(conn)`` instead) could pass every existing test while still +``physical_schema_of(conn)`` instead) could pass every existing test while still being broken for a real non-default schema. This exercises a representative subset of the refactored surface (FK trigger toggle, index create/drop, full-text install, sequence reset) against a genuine non-default Postgres diff --git a/tests/test_schema_reconcile.py b/tests/test_schema_reconcile.py index ef20b2f..d746d19 100644 --- a/tests/test_schema_reconcile.py +++ b/tests/test_schema_reconcile.py @@ -5,7 +5,7 @@ import sqlalchemy as sa from oa_configurator import ResolvedCDMDatabase, Role -from oa_configurator import qualified, schema_of, Dialect +from oa_configurator import qualified, physical_schema_of, Dialect from oa_configurator.testing import DIALECT_PARAMS, isolated_test_schema from omop_alchemy.backends.sqlite import SQLiteBackend from omop_alchemy.cdm.base.indexing import omop_index_name @@ -51,7 +51,7 @@ def reconcile_engine(request) -> _ReconcileEngine: if request.param == "postgresql": resolved = request.getfixturevalue("pg_db").resolved engine = request.getfixturevalue("pg_schema_session").get_bind() - schema = schema_of(engine) + schema = physical_schema_of(engine) resolved = dataclasses.replace( resolved, schema_name=schema, @@ -106,9 +106,9 @@ def test_reconcile_schema_reports_no_drift_on_fresh_database(reconcile_engine): def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(reconcile_engine): engine, resolved = reconcile_engine with engine.begin() as connection: - connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX, physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))}") + connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX, physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))}") connection.exec_driver_sql( - f"CREATE INDEX idx_gender ON {qualified(connection, 'person', physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))} (gender_concept_id)" + f"CREATE INDEX idx_gender ON {qualified(connection, 'person', physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))} (gender_concept_id)" ) report = reconcile_schema(engine, resolved=resolved) @@ -124,9 +124,9 @@ def test_reconcile_schema_reports_renamed_for_foreign_named_equivalent_index(rec def test_reconcile_schema_renamed_index_does_not_flip_table_to_drifted(reconcile_engine): engine, resolved = reconcile_engine with engine.begin() as connection: - connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX, physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))}") + connection.exec_driver_sql(f"DROP INDEX {qualified(connection, PERSON_GENDER_INDEX, physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))}") connection.exec_driver_sql( - f"CREATE INDEX idx_gender ON {qualified(connection, 'person', physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))} (gender_concept_id)" + f"CREATE INDEX idx_gender ON {qualified(connection, 'person', physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))} (gender_concept_id)" ) report = reconcile_schema(engine, resolved=resolved) @@ -242,9 +242,9 @@ def _index_issues(report): assert _index_issues(report) == [] with engine.begin() as connection: - connection.exec_driver_sql(f'DROP INDEX {qualified(connection, "ix_concept_concept_name_lower", physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))}') + connection.exec_driver_sql(f'DROP INDEX {qualified(connection, "ix_concept_concept_name_lower", physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))}') connection.exec_driver_sql( - f'CREATE INDEX ix_concept_concept_name_lower ON {qualified(connection, "concept", physical_schema=schema_of(connection, schema_tag=Role.PRIMARY))} (upper(concept_name))' + f'CREATE INDEX ix_concept_concept_name_lower ON {qualified(connection, "concept", physical_schema=physical_schema_of(connection, schema_tag=Role.PRIMARY))} (upper(concept_name))' ) report = reconcile_schema(engine, resolved=resolved, vocabulary_included=True) From e43ae90c9b1eb87985b14751e8c5844f293c763c Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Wed, 23 Sep 2026 05:29:34 +0000 Subject: [PATCH 29/32] Guard schema, remove old code, regression tests --- omop_alchemy/backends/base.py | 15 --- omop_alchemy/backends/postgres.py | 15 --- omop_alchemy/maintenance/cli_indexes.py | 88 ++++++++------ omop_alchemy/maintenance/cli_vocab.py | 32 +++-- tests/test_indexes.py | 1 + tests/test_load_vocab_source.py | 1 + tests/test_schema_provenance_guard.py | 154 +++++++++++++++++++++++- 7 files changed, 226 insertions(+), 80 deletions(-) diff --git a/omop_alchemy/backends/base.py b/omop_alchemy/backends/base.py index 49db32c..6de5c1e 100644 --- a/omop_alchemy/backends/base.py +++ b/omop_alchemy/backends/base.py @@ -149,21 +149,6 @@ def normalize_index_expression(self, sql_text: str) -> str: """ raise FeatureNotSupportedError("Functional-index expression normalization", self) - # ── Row counts ─────────────────────────────────────────────────────────── - - def approximate_row_counts( - self, - conn: sa.Connection, - schema: str, - ) -> dict[str, int]: - """Cheap, catalog-based row-count estimate per table in schema. - - Unlike most Backend methods, schema is explicit rather than read via - physical_schema_of(conn): a caller previewing an orphan schema is inspecting a - schema other than the connection's own configured one. - """ - raise FeatureNotSupportedError("Approximate row counts", self) - # ── Table operations ───────────────────────────────────────────────────── @abstractmethod diff --git a/omop_alchemy/backends/postgres.py b/omop_alchemy/backends/postgres.py index 4ee12b6..0c49933 100644 --- a/omop_alchemy/backends/postgres.py +++ b/omop_alchemy/backends/postgres.py @@ -164,21 +164,6 @@ def normalize_index_expression(self, sql_text: str) -> str: parts.append(_TEXTLIKE_CAST.sub("", sql_text[last_end:]).replace(" ", "").lower()) return "".join(parts) - # ── Row counts ─────────────────────────────────────────────────────────── - - def approximate_row_counts( - self, - conn: sa.Connection, - schema: str, - ) -> dict[str, int]: - rows = conn.execute( - sa.text( - "SELECT relname, n_live_tup FROM pg_stat_user_tables WHERE schemaname = :schema" - ), - {"schema": schema}, - ).all() - return {row.relname: row.n_live_tup for row in rows} - # ── Table operations ───────────────────────────────────────────────────── def analyze_table( diff --git a/omop_alchemy/maintenance/cli_indexes.py b/omop_alchemy/maintenance/cli_indexes.py index b4469af..c3babcb 100644 --- a/omop_alchemy/maintenance/cli_indexes.py +++ b/omop_alchemy/maintenance/cli_indexes.py @@ -700,7 +700,7 @@ def manage_indexes( results: list[IndexManagementResult] = [] with ExitStack() as guard_stack: - if enable and not dry_run: + if not dry_run: tables_by_schema_tag: dict[str, list[sa.Table]] = {} for table in selected_tables: tables_by_schema_tag.setdefault(table.schema_tag, []).append(table.table) @@ -1015,6 +1015,7 @@ def disable_indexes_command( enable=False, vocabulary_included=vocabulary_included, dry_run=dry_run, + resolved=conn.resolved, ) console.print(render_index_results(results)) console.print(render_index_summary(results, dry_run=dry_run)) @@ -1088,47 +1089,60 @@ def cluster_tables_command( selected_tables = select_omop_tables(vocabulary_included=vocabulary_included) results: list[IndexManagementResult] = [] - for table in selected_tables: - table_schema = physical_schema_of(engine, schema_tag=table.schema_tag) - if not inspector.has_table(table.table_name, schema=table_schema): - continue + with ExitStack() as guard_stack: + if not dry_run: + tables_by_schema_tag: dict[str, list[sa.Table]] = {} + for table in selected_tables: + tables_by_schema_tag.setdefault(table.schema_tag, []).append(table.table) + guard_connection = guard_stack.enter_context(engine.begin()) + # One provenance guard per schema_tag (count only known at runtime); ExitStack defers every write until the block below succeeds. + for schema_tag, tables in tables_by_schema_tag.items(): + # CLUSTER physically rewrites the table's heap: guard against a drifted schema the same as any other DDL. + guard_stack.enter_context( + guard_schema_provenance_for(guard_connection, conn.resolved, schema_tag=schema_tag, tables=tables) + ) - cluster_index_name = _cluster_target_name(table) - if cluster_index_name is None: - continue + for table in selected_tables: + table_schema = physical_schema_of(engine, schema_tag=table.schema_tag) + if not inspector.has_table(table.table_name, schema=table_schema): + continue - cluster_columns = _cluster_column_names(table, cluster_index_name) - existing_indexes = inspector.get_indexes(table.table_name, schema=table_schema) - physical_cluster_name = _resolve_physical_cluster_name( - existing_indexes, - cluster_index_name, - cluster_columns, - ) + cluster_index_name = _cluster_target_name(table) + if cluster_index_name is None: + continue - if not dry_run: - with engine.begin() as connection: - backend.cluster_table( - connection, table.table_name, physical_cluster_name, schema_tag=table.schema_tag + cluster_columns = _cluster_column_names(table, cluster_index_name) + existing_indexes = inspector.get_indexes(table.table_name, schema=table_schema) + physical_cluster_name = _resolve_physical_cluster_name( + existing_indexes, + cluster_index_name, + cluster_columns, + ) + + if not dry_run: + with engine.begin() as connection: + backend.cluster_table( + connection, table.table_name, physical_cluster_name, schema_tag=table.schema_tag + ) + with engine.connect() as connection: + backend.analyze_table(connection, table.table_name, schema_tag=table.schema_tag) + connection.commit() + + results.append( + IndexManagementResult( + operation="cluster", + table_name=table.table_name, + category=table.category, + schema_tag=table.schema_tag, + index_name=physical_cluster_name, + column_names=cluster_columns, + unique=False, + clustered=True, + enable=True, + status=dry_status(dry_run), + detail=dry_label(dry_run, "table would be clustered and analyzed", "table clustered and analyzed"), ) - with engine.connect() as connection: - backend.analyze_table(connection, table.table_name, schema_tag=table.schema_tag) - connection.commit() - - results.append( - IndexManagementResult( - operation="cluster", - table_name=table.table_name, - category=table.category, - schema_tag=table.schema_tag, - index_name=physical_cluster_name, - column_names=cluster_columns, - unique=False, - clustered=True, - enable=True, - status=dry_status(dry_run), - detail=dry_label(dry_run, "table would be clustered and analyzed", "table clustered and analyzed"), ) - ) console.print(render_index_results(results)) console.print(render_index_summary(results, dry_run=dry_run)) diff --git a/omop_alchemy/maintenance/cli_vocab.py b/omop_alchemy/maintenance/cli_vocab.py index 7604ccf..a335752 100644 --- a/omop_alchemy/maintenance/cli_vocab.py +++ b/omop_alchemy/maintenance/cli_vocab.py @@ -4,6 +4,7 @@ import time from collections.abc import Callable +from contextlib import ExitStack from dataclasses import dataclass from pathlib import Path from typing import Literal, TypeAlias, cast @@ -12,7 +13,7 @@ import sqlalchemy.orm as so from sqlalchemy.exc import OperationalError import typer -from oa_configurator import ensure_schema +from oa_configurator import ResolvedCDMDatabase, ensure_schema, guard_schema_provenance_for from orm_loader.backends import STAGING_SCHEMA, resolve_backend from orm_loader.helpers import Base from orm_loader.tables.typing import CSVTableProtocol @@ -241,6 +242,7 @@ def _create_missing_vocabulary_tables( connection: sa.Connection, *, db_schema: str | None, + resolved: ResolvedCDMDatabase | None = None, ) -> int: """Create any vocabulary-category ORM tables that are absent from the target database. Returns the count created.""" vocab_tables = select_maintenance_tables( @@ -255,11 +257,21 @@ def _create_missing_vocabulary_tables( if not missing_tables: return 0 - Base.metadata.create_all( - bind=connection, - tables=[table.table for table in missing_tables], - checkfirst=True, - ) + tables_by_schema_tag: dict[str, list[sa.Table]] = {} + for table in missing_tables: + tables_by_schema_tag.setdefault(table.schema_tag, []).append(table.table) + + with ExitStack() as guard_stack: + # One provenance guard per schema_tag (count only known at runtime); ExitStack defers every write until the block below succeeds. + for schema_tag, tables in tables_by_schema_tag.items(): + guard_stack.enter_context( + guard_schema_provenance_for(connection, resolved, schema_tag=schema_tag, tables=tables) + ) + Base.metadata.create_all( + bind=connection, + tables=[table.table for table in missing_tables], + checkfirst=True, + ) return len(missing_tables) @@ -278,6 +290,7 @@ def load_vocab_source( bulk_mode: bool = True, merge_batch_size: int | None = None, progress_callback: VocabularyLoadProgressCallback | None = None, + resolved: ResolvedCDMDatabase | None = None, ) -> VocabularyLoadReport: """ Load Athena vocabulary CSVs from source_path into the target database. @@ -302,6 +315,10 @@ def load_vocab_source( vocab_schema : str, optional Schema vocab-tagged tables live in, for the table-existence check against ``vocab_engine``. Defaults to ``db_schema``. + resolved : ResolvedCDMDatabase, optional + Enables the schema-provenance guard around any missing vocabulary + table creation. Omitted by direct test/programmatic callers with no + resolved config behind their engine, in which case the guard no-ops. """ vocab_engine = vocab_engine if vocab_engine is not None else engine vocab_schema = vocab_schema if vocab_schema is not None else db_schema @@ -408,7 +425,7 @@ def load_vocab_source( if not dry_run: with vocab_engine.connect() as pre_conn: created_table_count = _create_missing_vocabulary_tables( - pre_conn, db_schema=vocab_schema + pre_conn, db_schema=vocab_schema, resolved=resolved ) pre_conn.commit() @@ -719,6 +736,7 @@ def _update_progress(event: VocabularyLoadProgress) -> None: bulk_mode=bulk_mode, merge_batch_size=merge_batch_size, progress_callback=_update_progress, + resolved=conn.resolved, ) progress.update( task_id, completed=100.0, description="Athena vocabulary load complete" diff --git a/tests/test_indexes.py b/tests/test_indexes.py index c8c0568..9796dae 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -384,6 +384,7 @@ def fake_manage_indexes( db_schema: str | None = None, vocabulary_included: bool = False, dry_run: bool = False, + resolved: object = None, ) -> list[IndexManagementResult]: calls["engine"] = engine calls["enable"] = enable diff --git a/tests/test_load_vocab_source.py b/tests/test_load_vocab_source.py index 0c76cf7..eb83e83 100644 --- a/tests/test_load_vocab_source.py +++ b/tests/test_load_vocab_source.py @@ -201,6 +201,7 @@ def fake_load_vocab_source( bulk_mode: bool = True, merge_batch_size: int = 1_000_000, progress_callback=None, + resolved: object = None, # noqa: ARG001 ): calls["source_path"] = str(source_path) calls["dry_run"] = dry_run diff --git a/tests/test_schema_provenance_guard.py b/tests/test_schema_provenance_guard.py index 3e9120e..ad31ad1 100644 --- a/tests/test_schema_provenance_guard.py +++ b/tests/test_schema_provenance_guard.py @@ -1,13 +1,15 @@ -"""Schema-provenance guard wired into create_missing_tables(). +"""Schema-provenance guard wired into create_missing_tables(), install_fulltext_columns(), +manage_indexes(), and truncate_tables(). Live-Postgres regression: proves the guard actually fires and prevents DDL for a genuinely reconfigured schema, rather than passing by coincidence. -Only that one case is covered here. The guard's own agree/no-op/test_only +One case per call site is covered here. The guard's own agree/no-op/test_only semantics are already exhaustively covered at the primitive level in oa-configurator's own test suite; what's worth proving per consuming repo -is that this call site is actually wired to it, and a wiring mistake would -show up here too. +is that each call site is actually wired to it, and a wiring mistake (e.g. +a guard wrapping an empty `pass` instead of the real DDL) would show up here +too. pg_engine (unlike pg_db) is a real, committing engine, so every provenance row this file's tests write is a genuine commit. cleanup_after_test deletes @@ -27,15 +29,24 @@ from oa_configurator.testing import delete_rows_on_cleanup, isolated_test_schema +from omop_alchemy.backends import CONCEPT_NAME_TSVECTOR_COLUMN +from omop_alchemy.backends.base import FullTextError +from omop_alchemy.maintenance.cli_fulltext import install_fulltext_columns +from omop_alchemy.maintenance.cli_indexes import manage_indexes from omop_alchemy.maintenance.cli_schema_tables import create_missing_tables +from omop_alchemy.maintenance.cli_tables import truncate_tables +from omop_alchemy.maintenance.tables import TableCategory pytestmark = [pytest.mark.postgresql, pytest.mark.db_dialect] def _resolved(pg_db, *, database_name: str, schema: str): """pg_db.resolved with a unique name (the guard's own key includes it), - all three schemas pointed at schema, and connection.test_only forced - False so the guard doesn't no-op against pg_db's own test-only marking. + all three schemas pointed at schema, and both connection.test_only and + vocab_connection.test_only forced False so the guard doesn't no-op + against pg_db's own test-only marking. A VOCAB-tagged guard reads + vocab_connection specifically (connection_for_role(Role.VOCAB)), so + fixing only connection leaves it silently no-op'd. """ return dataclasses.replace( pg_db.resolved, @@ -44,6 +55,7 @@ def _resolved(pg_db, *, database_name: str, schema: str): vocab_schema=schema, results_schema=schema, connection=dataclasses.replace(pg_db.resolved.connection, test_only=False), + vocab_connection=dataclasses.replace(pg_db.resolved.vocab_connection, test_only=False), ) @@ -77,3 +89,133 @@ def test_create_missing_tables_guard_fires_on_reconfigured_schema(pg_db, pg_engi # The guard raises before create_all() runs: schema_b must still be empty. with engine_b.connect() as conn: assert sa.inspect(conn).get_table_names(schema=schema_b) == [] + + +def test_install_fulltext_columns_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_ft_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_ft_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + resolved_a = _resolved(pg_db, database_name=database_name, schema=schema_a) + # Guarded create establishes the provenance baseline for schema_a; an + # unguarded create here would leave install_fulltext_columns's own guard + # seeing "tables exist but no record", a false first-time-drift positive. + create_missing_tables(engine_a, vocabulary_included=True, resolved=resolved_a) + install_fulltext_columns(engine_a, resolved=resolved_a) + + engine_b = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} + ) + # install_fulltext_columns wraps every underlying error, including the + # guard's own SchemaDriftError, in its own FullTextError. + with pytest.raises(FullTextError) as exc_info: + install_fulltext_columns( + engine_b, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), + ) + assert isinstance(exc_info.value.__cause__, SchemaDriftError) + + # The guard raises before any ALTER TABLE runs: schema_b's concept table + # must not have picked up the tsvector sidecar column. + with engine_b.connect() as conn: + if sa.inspect(conn).has_table("concept", schema=schema_b): + columns = {c["name"] for c in sa.inspect(conn).get_columns("concept", schema=schema_b)} + assert CONCEPT_NAME_TSVECTOR_COLUMN not in columns + + +def test_manage_indexes_enable_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_idx_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_idx_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + resolved_a = _resolved(pg_db, database_name=database_name, schema=schema_a) + create_missing_tables(engine_a, vocabulary_included=True, resolved=resolved_a) + manage_indexes(engine_a, enable=True, cluster=False, resolved=resolved_a) + + engine_b = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} + ) + with pytest.raises(SchemaDriftError): + manage_indexes( + engine_b, + enable=True, + cluster=False, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), + ) + + +def test_manage_indexes_disable_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): + """Regression for the disable path specifically: manage_indexes(enable=False) + used to build no guard at all, regardless of resolved.""" + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_idxd_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_idxd_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + resolved_a = _resolved(pg_db, database_name=database_name, schema=schema_a) + create_missing_tables(engine_a, vocabulary_included=True, resolved=resolved_a) + manage_indexes(engine_a, enable=False, resolved=resolved_a) + + engine_b = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} + ) + create_missing_tables(engine_b, vocabulary_included=True) + with pytest.raises(SchemaDriftError): + manage_indexes( + engine_b, + enable=False, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), + ) + + +def test_truncate_tables_guard_fires_on_reconfigured_schema(pg_db, pg_engine, cleanup_after_test): + database_name = f"guard_wiring_test_db_{uuid.uuid4().hex[:8]}" + table = _schema_provenance_table(SCHEMA_PROVENANCE_SCHEMA) + delete_rows_on_cleanup( + cleanup_after_test, pg_engine, table, table.c.database_name == database_name + ) + with ( + isolated_test_schema(pg_engine, prefix="guard_wiring_trunc_a") as schema_a, + isolated_test_schema(pg_engine, prefix="guard_wiring_trunc_b") as schema_b, + ): + engine_a = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_a, "vocab": schema_a, "results": schema_a} + ) + resolved_a = _resolved(pg_db, database_name=database_name, schema=schema_a) + create_missing_tables(engine_a, vocabulary_included=True, resolved=resolved_a) + truncate_tables(engine_a, scope=TableCategory.VOCABULARY, cascade=True, resolved=resolved_a) + + engine_b = pg_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: schema_b, "vocab": schema_b, "results": schema_b} + ) + create_missing_tables(engine_b, vocabulary_included=True) + with pytest.raises(SchemaDriftError): + truncate_tables( + engine_b, + scope=TableCategory.VOCABULARY, + cascade=True, + resolved=_resolved(pg_db, database_name=database_name, schema=schema_b), + ) From ff6969d5a13393c2d9a86800d1d2f488a6bdc34e Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 24 Sep 2026 00:02:10 +0000 Subject: [PATCH 30/32] Correctly iterate over all schema tags instead of just roles --- omop_alchemy/maintenance/cli_schema_tables.py | 64 ++++++++++++------- omop_alchemy/maintenance/cli_tables.py | 38 ++++++++--- tests/conftest.py | 27 ++++---- tests/test_create_tables.py | 49 ++++++++++++++ 4 files changed, 132 insertions(+), 46 deletions(-) diff --git a/omop_alchemy/maintenance/cli_schema_tables.py b/omop_alchemy/maintenance/cli_schema_tables.py index ccf1d1e..1fef114 100644 --- a/omop_alchemy/maintenance/cli_schema_tables.py +++ b/omop_alchemy/maintenance/cli_schema_tables.py @@ -2,11 +2,20 @@ from __future__ import annotations +from collections.abc import Iterable +from contextlib import ExitStack from dataclasses import dataclass import sqlalchemy as sa -from oa_configurator import ResolvedCDMDatabase, Role, ensure_schema, guard_schema_provenance_for, physical_schema_of +from oa_configurator import ( + ResolvedCDMDatabase, + Role, + ensure_schema, + guard_schema_provenance_for, + physical_schema_of, + validate_schema_tag, +) from orm_loader.helpers import Base from ._cli_utils import Status, dry_label, dry_status from .tables import ( @@ -16,6 +25,11 @@ ) +def _distinct_schema_tags(tables: Iterable[sa.Table]) -> set[str]: + """Every distinct schema tag among tables. Skips untagged tables.""" + return {tag for table in tables if (tag := validate_schema_tag(table)) is not None} + + @dataclass(frozen=True) class TableCreationResult: """Outcome of attempting to create one missing ORM-managed table from SQLAlchemy metadata.""" @@ -77,19 +91,24 @@ def create_missing_tables( vocab_engine = vocab_engine if vocab_engine is not None else engine if not dry_run: ensure_schema(engine, physical_schema_of(engine, schema_tag=Role.PRIMARY)) - # create_all() would fail for non-existing vocab/results schemas on a fresh database + # create_all() would fail for a non-existing schema on a fresh database. if resolved is not None: - ensure_schema(engine, resolved.schema_for_role(Role.RESULTS)) - ensure_schema(vocab_engine, resolved.schema_for_role(Role.VOCAB)) + # Ensure schemas in split-engined deployments + for schema_tag in _distinct_schema_tags(Base.metadata.tables.values()): + # Primary schema is already ensure above + if schema_tag == Role.PRIMARY.value: + continue + target_engine = vocab_engine if schema_tag == Role.VOCAB.value else engine + ensure_schema(target_engine, physical_schema_of(target_engine, schema_tag=schema_tag)) inspector = sa.inspect(engine) missing_tables = collect_missing_tables( engine, vocabulary_included=vocabulary_included, ) - # Checking only primary schema would hide existing vocab/results tables, wrongly blocking dependents. + # Checking only primary schema would hide existing tables elsewhere, wrongly blocking dependents. existing_table_names: set[str] = set() - for role in Role: - existing_table_names |= set(inspector.get_table_names(schema=physical_schema_of(engine, schema_tag=role))) + for schema_tag in _distinct_schema_tags(Base.metadata.tables.values()): + existing_table_names |= set(inspector.get_table_names(schema=physical_schema_of(engine, schema_tag=schema_tag))) missing_table_names = {table.table_name for table in missing_tables} blocked_dependencies: dict[str, tuple[str, ...]] = {} @@ -112,32 +131,29 @@ def create_missing_tables( results: list[TableCreationResult] = [] if creatable_tables and not dry_run: all_tables = [table.table for table in creatable_tables] - tables_by_role = { - role: [table for table in all_tables if table.schema == role.value] - for role in Role - } + vocab_tables = [table for table in all_tables if table.schema == Role.VOCAB.value] + other_tables = [table for table in all_tables if table.schema != Role.VOCAB.value] if vocab_engine is engine: # One call: create_all's dependency sort and FK-deferral must see every table together. - with ( - engine.begin() as connection, - guard_schema_provenance_for(connection, resolved, schema_tag=Role.PRIMARY, tables=tables_by_role[Role.PRIMARY]), - guard_schema_provenance_for(connection, resolved, schema_tag=Role.RESULTS, tables=tables_by_role[Role.RESULTS]), - guard_schema_provenance_for(connection, resolved, schema_tag=Role.VOCAB, tables=tables_by_role[Role.VOCAB]), - ): + with engine.begin() as connection, ExitStack() as guards: + for schema_tag in _distinct_schema_tags(all_tables): + tables_for_tag = [table for table in all_tables if table.schema == schema_tag] + guards.enter_context( + guard_schema_provenance_for(connection, resolved, schema_tag=schema_tag, tables=tables_for_tag) + ) Base.metadata.create_all( bind=connection, tables=all_tables, checkfirst=True ) else: # Split physical connections: a cross-boundary FK can't be created here at all; # that failure surfaces from create_all itself rather than being masked. - vocab_tables = tables_by_role[Role.VOCAB] - other_tables = tables_by_role[Role.PRIMARY] + tables_by_role[Role.RESULTS] if other_tables: - with ( - engine.begin() as connection, - guard_schema_provenance_for(connection, resolved, schema_tag=Role.PRIMARY, tables=tables_by_role[Role.PRIMARY]), - guard_schema_provenance_for(connection, resolved, schema_tag=Role.RESULTS, tables=tables_by_role[Role.RESULTS]), - ): + with engine.begin() as connection, ExitStack() as guards: + for schema_tag in _distinct_schema_tags(other_tables): + tables_for_tag = [table for table in other_tables if table.schema == schema_tag] + guards.enter_context( + guard_schema_provenance_for(connection, resolved, schema_tag=schema_tag, tables=tables_for_tag) + ) Base.metadata.create_all( bind=connection, tables=other_tables, checkfirst=True ) diff --git a/omop_alchemy/maintenance/cli_tables.py b/omop_alchemy/maintenance/cli_tables.py index 959b014..e24678a 100644 --- a/omop_alchemy/maintenance/cli_tables.py +++ b/omop_alchemy/maintenance/cli_tables.py @@ -7,7 +7,15 @@ import sqlalchemy as sa import typer -from oa_configurator import ResolvedCDMDatabase, Role, autocommit_connection, guard_schema_provenance_for, qualified, physical_schema_of +from oa_configurator import ( + SCHEMA_TRANSLATE_MAP_KEY, + ResolvedCDMDatabase, + Role, + autocommit_connection, + guard_schema_provenance_for, + physical_schema_of, + qualified, +) from ..backends import resolve_backend, require_backend_support, backend_support_note from ._cli_utils import Status, dry_label, dry_status, omop_command, resolve_selection from .tables import ( @@ -119,6 +127,19 @@ class TruncateTableResult: detail: str +def _known_schema_tags(engine: sa.Engine) -> set[str]: + """Schema tags engine's own schema_translate_map actually routes, plus primary. + + Deliberately not oa_configurator.registered_schema_tags(): that's every tag + any package anywhere has registered, including ones for a wholly separate + database (e.g. omop_emb's "registry") that this engine has no entry for and + can't meaningfully resolve -- scanning those would check a schema that not + only doesn't exist but was never this engine's concern to begin with. + """ + stm = engine.get_execution_options().get(SCHEMA_TRANSLATE_MAP_KEY) or {} + return set(stm) | {Role.PRIMARY.value} + + def _blocking_foreign_key_references( engine: sa.Engine, inspector: sa.Inspector, @@ -127,19 +148,20 @@ def _blocking_foreign_key_references( ) -> dict[str, set[str]]: """Return tables outside the selection that FK-reference at least one selected table, preventing truncation. - A blocking table can live under any role's schema (a vocab table can - FK-reference a clinical table's PK, or vice versa), so every role's - schema is scanned, not just the primary one. + A blocking table can live under any schema tag engine itself routes (a + vocab table can FK-reference a clinical table's PK, or vice versa; + likewise an extension table), so every one of those is scanned, not + just the primary one. """ blockers: dict[str, set[str]] = {} - for role in Role: - role_schema = physical_schema_of(engine, schema_tag=role) - for table_name in inspector.get_table_names(schema=role_schema): + for schema_tag in _known_schema_tags(engine): + tag_schema = physical_schema_of(engine, schema_tag=schema_tag) + for table_name in inspector.get_table_names(schema=tag_schema): if table_name in selected_table_names: continue - for foreign_key in inspector.get_foreign_keys(table_name, schema=role_schema): + for foreign_key in inspector.get_foreign_keys(table_name, schema=tag_schema): referred_table = foreign_key.get("referred_table") if referred_table not in selected_table_names: continue diff --git a/tests/conftest.py b/tests/conftest.py index 1539e7a..776f9d1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -7,7 +7,7 @@ import typer.rich_utils as _typer_rich_utils from orm_loader.helpers import bootstrap from oa_configurator.testing import isolated_test_database, isolated_test_schema -from oa_configurator import SCHEMA_TRANSLATE_MAP_KEY, Role, ResolvedCDMDatabase, ResolvedConnection +from oa_configurator import ResolvedCDMDatabase, ResolvedConnection, registered_schema_tags import sqlalchemy.orm as so from sqlalchemy.orm import Session, sessionmaker @@ -71,15 +71,16 @@ def resolved_cdm_database_from_engine( def fresh_engine() -> Iterator[sa.Engine]: """Fresh, empty, function-scoped SQLite engine. - SQLite has no schema concept, so every schema tag maps back to None, matching - the flat namespace every caller here has always assumed. + SQLite has no schema concept, so every schema tag maps back to None, + matching the flat namespace every caller here has always assumed. + isolated_test_database's own default already folds every registered + tag this way when execution_options omits schema_translate_map. """ with isolated_test_database( OmopAlchemyConfig, "test_cdm_db_sqlite", dialect="sqlite", future=True, - execution_options={SCHEMA_TRANSLATE_MAP_KEY: {Role.PRIMARY.value: None, "vocab": None, "results": None}}, ) as db: yield db.connection.engine @@ -369,11 +370,9 @@ def engine(tmp_path_factory: pytest.TempPathFactory) -> Iterator[sa.Engine]: echo=False, poolclass=sa.pool.StaticPool, connect_args={"check_same_thread": False, "timeout": 30}, - # SQLite has no schema concept, and this fixture always represented - # a single flat namespace: map every schema tag back to None so the - # vocab/results-tagged tables land in the same place they always - # have here, unaffected by schema tagging. - execution_options={SCHEMA_TRANSLATE_MAP_KEY: {Role.PRIMARY.value: None, "vocab": None, "results": None}}, + # SQLite has no schema concept: isolated_test_database's own default + # folds every registered schema tag back to None, matching the flat + # namespace this fixture has always assumed. ) as db: engine = db.connection.engine bootstrap(engine, create=True) @@ -418,12 +417,12 @@ def pg_engine(pg_db): genuine engine-building code paths against (``.connect()``/``.begin()``, which a bare ``Connection`` can't stand in for). - A thin shim over ``pg_db``'s own ``committing_engine``: every schema tag - (``None``, ``"vocab"``, ``"results"``) folds back to the connection's - default, matching the single-schema setup ``pg_session`` provides. + A thin shim over ``pg_db``'s own ``committing_engine``: every registered + schema tag folds back to the connection's default, matching the + single-schema setup ``pg_session`` provides. """ return pg_db.committing_engine.execution_options( - schema_translate_map={Role.PRIMARY.value: None, "vocab": None, "results": None} + schema_translate_map={tag: None for tag in registered_schema_tags()} ) @@ -493,7 +492,7 @@ def pg_schema_session(pg_db): """ with isolated_test_schema(pg_db.committing_engine, prefix="omop_alchemy") as schema: engine = pg_db.committing_engine.execution_options( - schema_translate_map={Role.PRIMARY.value: schema, "vocab": schema, "results": schema} + schema_translate_map={tag: schema for tag in registered_schema_tags()} ) bootstrap(engine, create=True) session = so.Session(engine, expire_on_commit=False) diff --git a/tests/test_create_tables.py b/tests/test_create_tables.py index 4c7f4b5..c213fc0 100644 --- a/tests/test_create_tables.py +++ b/tests/test_create_tables.py @@ -1,6 +1,15 @@ +import dataclasses + import sqlalchemy as sa +import sqlalchemy.orm as so +from oa_configurator import Role, register_reserved_schema_tag +from orm_loader.helpers import Base +from omop_alchemy.maintenance import cli_schema_tables from omop_alchemy.maintenance.cli_schema import collect_missing_tables, create_missing_tables +from omop_alchemy.maintenance.tables import MaintenanceTable, TableCategory + +from tests.conftest import resolved_cdm_database_from_engine def test_collect_missing_tables_on_empty_database(fresh_engine): @@ -49,3 +58,43 @@ def test_create_missing_tables_can_create_vocabulary(fresh_engine): inspector = sa.inspect(fresh_engine) assert inspector.has_table("person") assert inspector.has_table("concept") + + +def test_create_missing_tables_creates_table_under_a_non_role_schema_tag(fresh_engine, monkeypatch): + """A table tagged with a registered schema tag outside Role is created, + not silently skipped by a Role-only enumeration. + + Uses a fake MaintenanceTable: tagging a real CDM table this way needs + extension-table support, not yet built on this branch. + """ + register_reserved_schema_tag("synthetic_tag", owner="test") + engine = fresh_engine.execution_options( + schema_translate_map={Role.PRIMARY.value: None, "vocab": None, "results": None, "synthetic_tag": None} + ) + resolved = resolved_cdm_database_from_engine(engine, name="synthetic_tag_test") + test_connection = dataclasses.replace(resolved.connection, test_only=True) + resolved = dataclasses.replace(resolved, connection=test_connection, vocab_connection=test_connection) + + class _SyntheticTagTable(Base): + __tablename__ = "synthetic_tag_table" + __table_args__ = {"schema": "synthetic_tag"} + id: so.Mapped[int] = so.mapped_column(primary_key=True) + + fake_table = MaintenanceTable( + table_name="synthetic_tag_table", + model_name="_SyntheticTagTable", + model_module=__name__, + category=TableCategory.METADATA, + table=_SyntheticTagTable.__table__, # ty: ignore[invalid-argument-type] + primary_key_columns=tuple(_SyntheticTagTable.__table__.primary_key.columns), # ty: ignore[unresolved-attribute] + ) + monkeypatch.setattr(cli_schema_tables, "collect_missing_tables", lambda *a, **k: [fake_table]) + + try: + results = create_missing_tables(engine, resolved=resolved) + result_by_name = {result.table_name: result for result in results} + assert result_by_name["synthetic_tag_table"].status == "created" + assert sa.inspect(engine).has_table("synthetic_tag_table") + finally: + Base.registry._dispose_cls(_SyntheticTagTable) + Base.metadata.remove(_SyntheticTagTable.__table__) # ty: ignore[invalid-argument-type] From a407d8328f780ef5479a7af3093fe951266ee1e1 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 24 Sep 2026 00:13:11 +0000 Subject: [PATCH 31/32] Docstring rectification --- omop_alchemy/cdm/base/concept_validation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/omop_alchemy/cdm/base/concept_validation.py b/omop_alchemy/cdm/base/concept_validation.py index 92ced1f..09ae95f 100644 --- a/omop_alchemy/cdm/base/concept_validation.py +++ b/omop_alchemy/cdm/base/concept_validation.py @@ -8,7 +8,9 @@ class ConceptValidationMixin: A concept-bearing column is defined as: - column name ends with '_concept_id' - - value is integer-like + - column name does not contain 'source' (excludes *_source_concept_id) + + No type check is performed; matching is by column-name pattern only. Works for: - ORM mapped tables From 98a6a0645b179044e0a7439279a35ba1dde53429 Mon Sep 17 00:00:00 2001 From: Nico Loesch Date: Thu, 24 Sep 2026 02:47:59 +0000 Subject: [PATCH 32/32] Corrected docs --- docs/advanced/backends.md | 41 ++++++++++++++++++++++++ docs/advanced/index.md | 1 + docs/cli/index.md | 4 +-- docs/getting-started/common-use-cases.md | 2 +- docs/getting-started/configuration.md | 2 +- docs/getting-started/index.md | 2 ++ docs/getting-started/maintenance.md | 10 +++++- docs/models/clinical/observation.md | 2 +- docs/models/index.md | 31 ++++++++++++++++++ docs/models/vocabulary/index.md | 2 +- docs/models/vocabulary/vocabulary.md | 2 +- docs/toolkit/analytics.md | 3 +- docs/toolkit/materialized-views.md | 2 +- 13 files changed, 93 insertions(+), 11 deletions(-) diff --git a/docs/advanced/backends.md b/docs/advanced/backends.md index e69de29..a42e9c0 100644 --- a/docs/advanced/backends.md +++ b/docs/advanced/backends.md @@ -0,0 +1,41 @@ +# Backend Compatibility + +Every maintenance operation goes through a `Backend` (`PostgresBackend` or +`SQLiteBackend`). PostgreSQL implements the full `Backend` interface. +SQLite implements only what SQLite itself can do; everything else raises +`FeatureNotSupportedError` at call time rather than failing silently or +producing a partial result. + +| Feature | PostgreSQL | SQLite | +| --- | --- | --- | +| Index existence check | Yes | Yes | +| Drop index if exists | Yes | Yes (unqualified — no schema concept) | +| `ANALYZE` | Yes | Yes | +| `VACUUM ANALYZE` | Yes | No | +| FK trigger management / status | Yes | No | +| FK constraint violation counting | Yes | No | +| Table clustering (`CLUSTER`) | Yes | No | +| Cluster index inspection | Yes | No | +| Functional-index expression normalization | Yes | No (SQLite never reflects expression-based indexes) | +| `TRUNCATE ... RESTART IDENTITY / CASCADE` | Yes | No | +| Sequence lookup / reset | Yes | No (SQLite has no sequences) | +| Full-text search | Yes — see [PostgreSQL Full-Text Search](fulltext.md) | No | +| Database backup / restore | Yes | No | + +## What this means in practice + +Maintenance CLI commands that rely on a not-supported feature raise +`FeatureNotSupportedError` on SQLite rather than doing nothing. In +particular, against a SQLite database: + +- `indexes cluster` and the clustering step of `manage_indexes --enable` + are unavailable. +- `truncate-tables` cannot use `RESTART IDENTITY`/`CASCADE`. +- `fulltext install` is unavailable entirely. +- `backup-database`/`restore-database` are unavailable entirely. +- FK trigger toggling and FK violation counting are unavailable. + +SQLite remains fully supported for the core ORM/CDM layer (models, queries, +`create_missing_tables`, schema-provenance guarding) — these limitations are +specific to the maintenance operations listed above, which assume a +PostgreSQL-grade catalog. diff --git a/docs/advanced/index.md b/docs/advanced/index.md index c2f08d5..976c809 100644 --- a/docs/advanced/index.md +++ b/docs/advanced/index.md @@ -17,3 +17,4 @@ the immutability and interpretability of the underlying CDM tables. - [Backend Compatibility](backends.md) - [PostgreSQL Full-Text Search](fulltext.md) +- [Vocabulary Load Performance](vocabulary_load_performance.md) diff --git a/docs/cli/index.md b/docs/cli/index.md index 5548371..f64bdb0 100644 --- a/docs/cli/index.md +++ b/docs/cli/index.md @@ -90,7 +90,7 @@ With the decorator, the function body is all that matters: @app.command("my-command") @omop_command("my-command") def my_command(conn, engine) -> None: - results = do_work(engine, db_schema=conn.db_schema) + results = do_work(engine, db_schema=conn.resolved.schema_name) console.print(render_results(results)) ``` @@ -102,5 +102,5 @@ def my_command(conn, engine) -> None: | Attribute | Description | |---|---| -| `conn.db_schema` | CDM schema name from the resolved database (e.g. `"omop"`) | +| `conn.resolved` | The resolved `ResolvedCDMDatabase`; `conn.resolved.schema_name` is the CDM schema name (e.g. `"omop"`) | | `conn.athena_source` | Athena vocabulary CSV directory from `[tools.omop_alchemy]`'s `athena_source_path` field; `None` if not configured | diff --git a/docs/getting-started/common-use-cases.md b/docs/getting-started/common-use-cases.md index ba14b08..a729a7f 100644 --- a/docs/getting-started/common-use-cases.md +++ b/docs/getting-started/common-use-cases.md @@ -117,7 +117,7 @@ vocab_connection = "vocab" # <- references your vocabulary DB - This is **NEVER** done automatically to preserve data integrity from our end. 3. Record the new schema as the accepted baseline following the assumptions listed in "Scenario" above: ```bash - omop-config acknowledge-schema-migration --database my_db --role vocab --new-schema myvocab --reason "moving vocab off the shared schema" + omop-config acknowledge-schema-migration --database my_db --schema-tag vocab --new-schema myvocab --reason "moving vocab off the shared schema" ``` 4. Once you've confirmed the new schema is correct, clean up the old one: ```bash diff --git a/docs/getting-started/configuration.md b/docs/getting-started/configuration.md index 8d566b6..8fc07bf 100644 --- a/docs/getting-started/configuration.md +++ b/docs/getting-started/configuration.md @@ -51,7 +51,7 @@ categories: - `concept`, `concept_ancestor`, `concept_class`, `concept_relationship`, `concept_synonym`, `domain`, `drug_strength`, `relationship`, `source_to_concept_map`, `vocabulary` - Controlled by `vocab_schema` in the configuration. - **Results/analytics tables** (`Role.RESULTS`): - - `cohort`, `cohort_Definition` + - `cohort`, `cohort_definition` - Controlled by `results_schema` in the configuration. ![OMOP CDM v5.4](https://ohdsi.github.io/CommonDataModel/man/images/cdm55.png) diff --git a/docs/getting-started/index.md b/docs/getting-started/index.md index 39c58f6..c085980 100644 --- a/docs/getting-started/index.md +++ b/docs/getting-started/index.md @@ -9,5 +9,7 @@ These pages cover installation, maintenance tooling, and a minimal quickstart fo ## Orientation - [Installation](installation.md) +- [Configuration](configuration.md) - [Maintenance CLI](maintenance.md) +- [Common Use Cases](common-use-cases.md) - [Quickstart](quickstart.md) diff --git a/docs/getting-started/maintenance.md b/docs/getting-started/maintenance.md index 336b174..0b4ed88 100644 --- a/docs/getting-started/maintenance.md +++ b/docs/getting-started/maintenance.md @@ -16,7 +16,7 @@ Some commands depend on PostgreSQL-specific features and will return an error if | Command group | Requires PostgreSQL | Why | | --- | --- | --- | -| `load-vocab-source` | No (PostgreSQL + SQLite) | Uses ORM CSV loader; `--bulk-mode` and `--db-schema` are PostgreSQL-only | +| `load-vocab-source` | No (PostgreSQL + SQLite) | Uses ORM CSV loader; `--bulk-mode` is PostgreSQL-only | | `indexes` | No (cluster apply is PostgreSQL-only) | Index DDL is standard SQL; `CLUSTER` is PostgreSQL | | `create-missing-tables`, `reconcile-schema`, `data-summary`, `info`, `doctor` | No | Pure SQLAlchemy metadata operations | | `reset-sequences` | Yes | PostgreSQL sequences (`SETVAL`) | @@ -271,6 +271,13 @@ omop-alchemy reset-sequences --vocab # vocabulary tables only --- +!!! note "Schema-drift protection" + `fulltext install`, `indexes enable`/`disable`/`cluster`, `truncate-tables`, and + vocabulary-table creation each guard their DDL against the configured schema having + silently drifted since it was last recorded, and raise `SchemaDriftError` if it has. + See [Schema drift](#schema-drift) below for the remediation path + (`omop-config acknowledge-schema-migration`). + ## Command reference | Command | Purpose | Key options | Backend | @@ -290,6 +297,7 @@ omop-alchemy reset-sequences --vocab # vocabulary tables only | `analyze-tables` | Refresh planner statistics | `--scope`, `--table`, `--vacuum` | PostgreSQL, SQLite (`--vacuum` PostgreSQL-only) | | `indexes disable` | Drop ORM-defined secondary indexes | `--vocab`, `--dry-run` | All | | `indexes enable` | Recreate ORM-defined secondary indexes | `--vocab`, `--dry-run` | All (cluster on PostgreSQL) | +| `indexes cluster` | Physically rewrite tables sorted by their cluster index | `--vocab`, `--dry-run` | PostgreSQL | | `fulltext install` | Add tsvector sidecar columns to vocabulary tables | `--regconfig`, `--no-create-indexes` | PostgreSQL | | `fulltext populate` | Populate sidecar tsvector vectors | `--regconfig` | PostgreSQL | | `fulltext drop` | Remove tsvector sidecar columns and indexes | | PostgreSQL | diff --git a/docs/models/clinical/observation.md b/docs/models/clinical/observation.md index 809e8aa..6dcb23d 100644 --- a/docs/models/clinical/observation.md +++ b/docs/models/clinical/observation.md @@ -1,4 +1,4 @@ -# oubservation +# observation > Documentation coming soon. diff --git a/docs/models/index.md b/docs/models/index.md index c6563dd..0ba0194 100644 --- a/docs/models/index.md +++ b/docs/models/index.md @@ -43,3 +43,34 @@ validation, and reuse. - [Drug Era](derived/drug_era.md) - [Dose Era](derived/dose_era.md) - [Cohort & Cohort Definitions](derived/cohort.md) + +--- + +## Health Economic + +- [Cost](health_economic/cost.md) +- [Payer Plan Period](health_economic/payer_plan_period.md) + +--- + +## Metadata + +- [CDM Source](metadata/cdm_source.md) +- [Metadata](metadata/metadata.md) + +--- + +## Structural + +- [Episode](structural/episode.md) +- [Episode Event](structural/episode_event.md) +- [Fact Relationship](structural/fact_relationship.md) + +--- + +## Unstructured + +- [Note](unstructured/note.md) +- [Note NLP](unstructured/note_nlp.md) +- [Image](unstructured/image.md) +- [Image Feature](unstructured/image_feature.md) diff --git a/docs/models/vocabulary/index.md b/docs/models/vocabulary/index.md index bea4cf8..72a8586 100644 --- a/docs/models/vocabulary/index.md +++ b/docs/models/vocabulary/index.md @@ -1,4 +1,4 @@ -# vocaubulary Models +# vocabulary Models This section contains ORM models corresponding to OMOP CDM vocabulary tables. diff --git a/docs/models/vocabulary/vocabulary.md b/docs/models/vocabulary/vocabulary.md index 321f8c8..420c6b9 100644 --- a/docs/models/vocabulary/vocabulary.md +++ b/docs/models/vocabulary/vocabulary.md @@ -1,4 +1,4 @@ -# vocaubulary +# vocabulary > Documentation coming soon. diff --git a/docs/toolkit/analytics.md b/docs/toolkit/analytics.md index d7c0f67..8967cac 100644 --- a/docs/toolkit/analytics.md +++ b/docs/toolkit/analytics.md @@ -82,8 +82,7 @@ The single-value properties return the first modality in this order for which th The oncology package publishes lazy governed concept specifications for T, N, M, and group stage, tumour grade, and metastatic disease. Laterality and tumour size are exposed as governed scalar accessors. These declarations consume -`omop-semantics`; the narrow metastatic-disease descendant group requires -`omop-semantics` 0.6.1. Importing the module does not expand a vocabulary or +`omop-semantics` (the package already requires `>=0.6.2`). Importing the module does not expand a vocabulary or contact a database. Stage selection is a query policy over an already filtered canonical modifier diff --git a/docs/toolkit/materialized-views.md b/docs/toolkit/materialized-views.md index f2c828a..13da8ca 100644 --- a/docs/toolkit/materialized-views.md +++ b/docs/toolkit/materialized-views.md @@ -4,7 +4,7 @@ A materialized view is a persisted read model: an expensive or carefully defined ## The deployment contract -The supported deployment contract is PostgreSQL with unqualified materialized-view names resolving to the `public` schema through the connection's `search_path`. Leave `schema` at its default when calling the lifecycle methods. Qualified non-`public` schemas and schema translation are not part of this package's materialized-view contract. +The supported deployment contract is PostgreSQL with unqualified materialized-view names resolving to the `public` schema through the connection's `search_path`. Leave `schema_tag` at its default when calling the lifecycle methods. Qualified non-`public` schemas and schema translation are not part of this package's materialized-view contract. OMOP Alchemy owns the OMOP models and the query-building vocabulary. [`orm-loader`](https://australiancancerdatanetwork.github.io/orm-loader/tables/mat_view/) owns the generic materialized-view definition and database lifecycle. Keeping that boundary means this package can describe an OMOP read model without growing a second implementation of DDL, refresh, index creation, or dependency ordering.