diff --git a/.gitignore b/.gitignore index fc89a057..92920303 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ not_for_github/ cdl_skill/ .DS_Store +mutants/ # git worktree .history diff --git a/docs/schema_converters.md b/docs/schema_converters.md index 08f21931..1879ad05 100644 --- a/docs/schema_converters.md +++ b/docs/schema_converters.md @@ -211,8 +211,8 @@ from cdm_data_loaders.converters.facades.jsonschema_to_pyarrow import JSONSchema from cdm_data_loaders.converters.sources.json_schema.permissive import JsonSchemaReader KBASE_EXTENSION_SPEC: Final[ExtensionSpec] = ExtensionSpec( - "x-kbase", - { + namespace="x-kbase", + schema={ "type": "object", "properties": { "ontology_term": {"type": "string"}, diff --git a/docs/uniprot.md b/docs/uniprot.md index af01d48e..1aa56000 100644 --- a/docs/uniprot.md +++ b/docs/uniprot.md @@ -2,16 +2,16 @@ `cdm-data-loaders` provides `dlt` pipelines for ingesting UniProt Knowledgebase (UniProtKB) and UniRef XML datasets (`*.xml` and `*.xml.gz`) as well as tabular UniProt ID mapping releases (`*.dat` and `*.dat.gz`), along with parsers for UniProt release metadata. -| Pipeline CLI | Mode | Source Module | Bundled Schema / Dataset Module | Default Dataset / Table | -| :------------------ | :---------------------------- | :---------------------------------------------------- | :-------------------------------------------- | :--------------------------------------------------- | -| `uniprot` | XSD-to-PyArrow (single table) | `cdm_data_loaders.pipelines.presets.uniprot_xml` | `cdm_data_loaders/schemas/uniprot/uniprot.xsd` | `uniprot` / `entry` | -| `uniref50` | XSD-to-PyArrow (single table) | `cdm_data_loaders.pipelines.presets.uniprot_xml` | `cdm_data_loaders/schemas/uniref/uniref.xsd` | `uniref50` / `entry` | -| `uniref90` | XSD-to-PyArrow (single table) | `cdm_data_loaders.pipelines.presets.uniprot_xml` | `cdm_data_loaders/schemas/uniref/uniref.xsd` | `uniref90` / `entry` | -| `uniref100` | XSD-to-PyArrow (single table) | `cdm_data_loaders.pipelines.presets.uniprot_xml` | `cdm_data_loaders/schemas/uniref/uniref.xsd` | `uniref100` / `entry` | -| `uniprot_cdm` | CDM-normalized (multi-table) | `cdm_data_loaders.pipelines.presets.uniprot_cdm` | `cdm_data_loaders.datasets.uniprot.kb` | `uniprot_cdm` (multi-table) | -| `uniref50_cdm` | CDM-normalized (multi-table) | `cdm_data_loaders.pipelines.presets.uniprot_cdm` | `cdm_data_loaders.datasets.uniprot.uniref` | `uniref50_cdm` (multi-table) | -| `uniref90_cdm` | CDM-normalized (multi-table) | `cdm_data_loaders.pipelines.presets.uniprot_cdm` | `cdm_data_loaders.datasets.uniprot.uniref` | `uniref90_cdm` (multi-table) | -| `uniref100_cdm` | CDM-normalized (multi-table) | `cdm_data_loaders.pipelines.presets.uniprot_cdm` | `cdm_data_loaders.datasets.uniprot.uniref` | `uniref100_cdm` (multi-table) | +| Pipeline CLI | Mode | Source Module | Bundled Schema / Dataset Module | Default Dataset / Table | +| :------------------ | :---------------------------- | :----------------------------------------------------- | :--------------------------------------------- | :--------------------------------------------------- | +| `uniprot` | XSD-to-PyArrow (single table) | `cdm_data_loaders.pipelines.presets.uniprot_xml` | `cdm_data_loaders/schemas/uniprot/uniprot.xsd` | `uniprot` / `entry` | +| `uniref50` | XSD-to-PyArrow (single table) | `cdm_data_loaders.pipelines.presets.uniprot_xml` | `cdm_data_loaders/schemas/uniref/uniref.xsd` | `uniref50` / `entry` | +| `uniref90` | XSD-to-PyArrow (single table) | `cdm_data_loaders.pipelines.presets.uniprot_xml` | `cdm_data_loaders/schemas/uniref/uniref.xsd` | `uniref90` / `entry` | +| `uniref100` | XSD-to-PyArrow (single table) | `cdm_data_loaders.pipelines.presets.uniprot_xml` | `cdm_data_loaders/schemas/uniref/uniref.xsd` | `uniref100` / `entry` | +| `uniprot_cdm` | CDM-normalized (multi-table) | `cdm_data_loaders.pipelines.presets.uniprot_cdm` | `cdm_data_loaders.datasets.uniprot.uniprot_kb` | `uniprot_cdm` (multi-table) | +| `uniref50_cdm` | CDM-normalized (multi-table) | `cdm_data_loaders.pipelines.presets.uniprot_cdm` | `cdm_data_loaders.datasets.uniprot.uniref` | `uniref50_cdm` (multi-table) | +| `uniref90_cdm` | CDM-normalized (multi-table) | `cdm_data_loaders.pipelines.presets.uniprot_cdm` | `cdm_data_loaders.datasets.uniprot.uniref` | `uniref90_cdm` (multi-table) | +| `uniref100_cdm` | CDM-normalized (multi-table) | `cdm_data_loaders.pipelines.presets.uniprot_cdm` | `cdm_data_loaders.datasets.uniprot.uniref` | `uniref100_cdm` (multi-table) | | `uniprot_idmapping` | XSV preset (dual-table) | `cdm_data_loaders.pipelines.presets.uniprot_idmapping` | `cdm_data_loaders.datasets.uniprot.idmapping` | `uniprot_idmapping` (`raw_id_mapping`, `id_mapping`) | --- @@ -45,7 +45,7 @@ When loading directly into the KBase Common Data Model (CDM) relational table st ### `uniprot_cdm` -[`uniprot_cdm`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py) converts each `{https://uniprot.org/uniprot}entry` element against [`uniprot.xsd`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/schemas/uniprot/uniprot.xsd) and routes rows via [`datasets/uniprot/kb.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/datasets/uniprot/kb.py) into normalized tables under dataset `uniprot_cdm`: +[`uniprot_cdm`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py) converts each `{https://uniprot.org/uniprot}entry` element against [`uniprot.xsd`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/schemas/uniprot/uniprot.xsd) and routes rows via [`datasets/uniprot/uniprot_kb.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/datasets/uniprot/uniprot_kb.py) into normalized tables under dataset `uniprot_cdm`: | Table Name | Contents | | :--------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | diff --git a/pyproject.toml b/pyproject.toml index 3a8f068c..43914c81 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -233,8 +233,9 @@ build-backend = "uv_build" [tool.mutmut] source_paths = [ "src/" ] -pytest_add_cli_args_test_selection= [ "-m", "not requires_spark", "-m", "not slow_test", "-m", "not requires_ceph", "-m", "not external_request", "-m", "not docker", "-m", "not docs", "-m", "not state_audit" ] - +pytest_add_cli_args_test_selection= [ "-m", "not requires_spark", "-m", "not slow_test", "-m", "not requires_ceph", "-m", "not external_request", "-m", "not docker", "-m", "not state_audit", "tests/cdm_data_loaders/"] +also_copy = ["docs/", "docs_macros/", "scripts/", "specs/", "notebooks/"] +do_not_mutate = [ "src/cdm_data_loaders/schemas/**/schema.py"] [tool.pytest] diff --git a/specs/_loader/converter_spec.py b/specs/_loader/converter_spec.py index 1b32ce50..4abf0b21 100644 --- a/specs/_loader/converter_spec.py +++ b/specs/_loader/converter_spec.py @@ -9,26 +9,29 @@ The same manifests drive the specification tests and the generated specification documents. """ -from dataclasses import dataclass, field from pathlib import Path from typing import Any +from pydantic import BaseModel, ConfigDict, Field + from specs._loader.yaml_io import CASE_MANIFEST, load_yaml_dict -@dataclass(frozen=True, slots=True) -class Rule: +class Rule(BaseModel): """A single conversion or pipeline specification rule.""" + model_config = ConfigDict(extra="forbid", frozen=True) + id: str statement: str section_id: str -@dataclass(frozen=True, slots=True) -class Section: +class Section(BaseModel): """A group of rules rendered under one heading.""" + model_config = ConfigDict(extra="forbid", frozen=True) + id: str title: str intro: str @@ -36,14 +39,15 @@ class Section: rules: tuple[Rule, ...] -@dataclass(frozen=True, slots=True) -class RuleCatalogue: +class RuleCatalogue(BaseModel): """All rules and front-matter metadata for one converter or pipeline specification.""" + model_config = ConfigDict(extra="forbid", frozen=True) + title: str intro: str sections: tuple[Section, ...] - metadata: dict[str, Any] = field(default_factory=dict) + metadata: dict[str, Any] = Field(default_factory=dict) @property def rules(self) -> dict[str, Rule]: @@ -55,38 +59,49 @@ def rules(self) -> dict[str, Rule]: return {rule.id: rule for section in self.sections for rule in section.rules} -@dataclass(frozen=True, slots=True) -class Run: +class Run(BaseModel): """One execution of a converter case: input data, parameters, and expected outputs.""" + model_config = ConfigDict(extra="forbid", frozen=True) + id: str data: Path expected_json: Path expected_table: Path xml_tag: str | None - params: dict[str, Any] = field(default_factory=dict) + params: dict[str, Any] = Field(default_factory=dict) schema_attr: str = "EXPECTED_SCHEMA" table_attr: str = "EXPECTED_TABLE" expected_count: int | None = None expected_rejected: Path | None = None - extra_expected_json: dict[str, Path] = field(default_factory=dict) + extra_expected_json: dict[str, Path] = Field(default_factory=dict) -@dataclass(frozen=True, slots=True) -class Case: +class Case(BaseModel): """A converter fixture directory with its manifest.""" + model_config = ConfigDict(extra="forbid", frozen=True, validate_by_name=True, validate_by_alias=True) + suite: str name: str directory: Path title: str - schema: Path + schema_path: Path = Field(alias="schema") expected_schema: Path xml_tag: str | None rules: tuple[str, ...] notes: str runs: tuple[Run, ...] + @property + def schema(self) -> Path: # type: ignore[override] + """Return the input schema file path for this converter case. + + :return: Resolved input schema path. + :rtype: Path + """ + return self.schema_path + @property def anchor(self) -> str: """Return the stable HTML anchor for the case. @@ -97,16 +112,17 @@ def anchor(self) -> str: return f"case-{self.suite}-{self.name}".replace("_", "-").lower() -@dataclass(frozen=True, slots=True) -class ErrorCase: +class ErrorCase(BaseModel): """An input that must be rejected with a specific error code.""" + model_config = ConfigDict(extra="forbid", frozen=True) + file: Path code: str match: str description: str xml_tag: str | None - params: dict[str, Any] = field(default_factory=dict) + params: dict[str, Any] = Field(default_factory=dict) def load_rules(path: Path) -> RuleCatalogue: diff --git a/specs/_loader/dlt_contracts.py b/specs/_loader/dlt_contracts.py index fe2d74ae..2881ee5c 100644 --- a/specs/_loader/dlt_contracts.py +++ b/specs/_loader/dlt_contracts.py @@ -1,6 +1,5 @@ """Static PyArrow probe schemas and dlt normalization contracts shared by docs macros and tests.""" -from dataclasses import dataclass from datetime import UTC, date, datetime from decimal import Decimal from enum import StrEnum @@ -8,6 +7,7 @@ import pyarrow as pa from frozendict import frozendict +from pydantic import BaseModel, ConfigDict from pyiceberg.schema import Schema as IcebergSchema from pyiceberg.types import ( BooleanType, @@ -845,10 +845,11 @@ class TableShape(StrEnum): ) -@dataclass(frozen=True, slots=True) -class ShapeExpectation: +class ShapeExpectation(BaseModel): """Input schema, rows, and per-layer expectations for one ``TableShape``.""" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) + input_arrow_schema: pa.Schema table_metadata: dict[str, str] rows: tuple[dict[str, Any], ...] diff --git a/specs/_loader/pipeline_spec.py b/specs/_loader/pipeline_spec.py index 7644f54c..b1e68a7c 100644 --- a/specs/_loader/pipeline_spec.py +++ b/specs/_loader/pipeline_spec.py @@ -2,11 +2,11 @@ import json import sys -from dataclasses import dataclass, field from pathlib import Path from typing import Any, Final import pyarrow as pa +from pydantic import BaseModel, ConfigDict, Field from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA from specs._loader.yaml_io import CASE_MANIFEST, load_settings_file, load_spec_module, load_yaml_dict @@ -14,40 +14,43 @@ RELATIVE_PATH_SETTINGS_KEYS: Final[frozenset[str]] = frozenset({"xsd_file", "schema_file", "log_config_file"}) -@dataclass(frozen=True, slots=True) -class ExpectedReport: +class ExpectedReport(BaseModel): """Expected contents of ``report.txt`` written by ``save_pipeline_metadata``.""" + model_config = ConfigDict(extra="forbid", frozen=True) + status: str = "completed" ingested_files: tuple[str, ...] = () load_errors: tuple[str, ...] = () pipeline_load_info_errs: str | None = None -@dataclass(frozen=True, slots=True) -class PipelineRun: +class PipelineRun(BaseModel): """One execution of a pipeline case: input path, settings, and expected outputs.""" + model_config = ConfigDict(extra="forbid", frozen=True) + id: str input_dir: Path input_subdir: str | None = None settings_file: Path | None = None - settings: dict[str, Any] = field(default_factory=dict) - expected_report: ExpectedReport = field(default_factory=ExpectedReport) - expected_counts: dict[str, int] = field(default_factory=dict) + settings: dict[str, Any] = Field(default_factory=dict) + expected_report: ExpectedReport = Field(default_factory=ExpectedReport) + expected_counts: dict[str, int] = Field(default_factory=dict) expected_tables: Path | None = None expected_json: Path | None = None tables_attr: str = "EXPECTED_TABLES" primary_table: str | None = None expected_rejected: Path | None = None - extra_expected_json: dict[str, Path] = field(default_factory=dict) + extra_expected_json: dict[str, Path] = Field(default_factory=dict) smoke: bool = True -@dataclass(frozen=True, slots=True) -class PipelineCase: +class PipelineCase(BaseModel): """A pipeline specification fixture directory with its manifest.""" + model_config = ConfigDict(extra="forbid", frozen=True) + pipeline: str name: str directory: Path @@ -219,10 +222,11 @@ def load_expected_json_tables(run: PipelineRun) -> dict[str, list[dict[str, Any] return result -@dataclass(frozen=True, slots=True) -class _CaseRunContext: +class _CaseRunContext(BaseModel): """Case-level defaults shared across all runs in one ``case.yaml`` manifest.""" + model_config = ConfigDict(extra="forbid", frozen=True) + pipeline: str directory: Path case_raw: dict[str, Any] diff --git a/src/cdm_data_loaders/converters/core/dlt_normalization.py b/src/cdm_data_loaders/converters/core/dlt_normalization.py index c123e9af..36fe000c 100644 --- a/src/cdm_data_loaders/converters/core/dlt_normalization.py +++ b/src/cdm_data_loaders/converters/core/dlt_normalization.py @@ -3,21 +3,29 @@ from collections import deque from collections.abc import Mapping from copy import deepcopy -from dataclasses import dataclass, field -from typing import Final, cast +from dataclasses import field +from typing import Annotated, Final, cast from dlt.common.schema.typing import TTableSchema +from pydantic import ConfigDict, GetPydanticSchema, InstanceOf, StrictStr +from pydantic.dataclasses import dataclass, rebuild_dataclass +from pydantic_core import core_schema from cdm_data_loaders.converters.core.errors import ConversionError NESTED_TABLE_SEPARATOR: Final[str] = "__" +type _TableSchemaField = Annotated[ + TTableSchema, + GetPydanticSchema(lambda _tp, _handler: core_schema.is_instance_schema(Mapping)), +] + class DltNormalizationError(ConversionError): """Raised for malformed tables, invalid parent graphs, or inconsistent trees.""" -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True, slots=True, config=ConfigDict(extra="forbid")) class DltTableNode: """A table with its mapping name, declared parent, property key, and raw definition. @@ -26,11 +34,14 @@ class DltTableNode: Identity fields are frozen; table and children mappings are independently owned. """ - name: str - parent: str | None - key: str - table: TTableSchema - children: dict[str, "DltTableNode"] = field(default_factory=dict) + name: StrictStr + parent: StrictStr | None + key: StrictStr + table: _TableSchemaField + children: dict[StrictStr, InstanceOf["DltTableNode"]] = field(default_factory=dict) + + +rebuild_dataclass(DltTableNode) def child_key(parent_name: str, child_name: str) -> str: diff --git a/src/cdm_data_loaders/converters/core/errors.py b/src/cdm_data_loaders/converters/core/errors.py index c2058a3c..81d2ad01 100644 --- a/src/cdm_data_loaders/converters/core/errors.py +++ b/src/cdm_data_loaders/converters/core/errors.py @@ -1,9 +1,11 @@ """Shared error types and helpers for the converter packages.""" from collections.abc import Mapping -from typing import ClassVar, NoReturn +from typing import Annotated, ClassVar, NoReturn from frozendict import frozendict +from pydantic import AfterValidator, ValidationError +from pydantic.dataclasses import rebuild_dataclass class ConversionError(ValueError): @@ -13,6 +15,22 @@ class ConversionError(ValueError): title: ClassVar[str] = "" +def _freeze_error_by_code( + value: Mapping[str, type[ConversionError]], +) -> frozendict[str, type[ConversionError]]: + """Freeze a validated error-code mapping into an immutable ``frozendict``. + + :param value: Validated mapping from specification code to exception class. + :type value: Mapping[str, type[ConversionError]] + :return: Immutable ``frozendict`` copy. + :rtype: frozendict[str, type[ConversionError]] + """ + return frozendict(value) + + +type ErrorCodeMap = Annotated[Mapping[str, type[ConversionError]], AfterValidator(_freeze_error_by_code)] + + def build_error_by_code[E: ConversionError](errors: tuple[type[E], ...]) -> frozendict[str, type[E]]: """Build an immutable mapping from specification ``code`` to exception class. @@ -50,3 +68,33 @@ def raise_spec_error( if cause is not None: raise cls(msg) from cause raise cls(msg) + + +def unwrap_conversion_errors[ModelT](cls: type[ModelT]) -> type[ModelT]: + """Re-raise sole ``ConversionError`` causes of Pydantic ``ValidationError`` unwrapped. + + Pydantic dataclasses wrap ``ValueError`` subclasses (including + ``ConversionError``) raised inside validators into ``ValidationError``. + Rebuilding the dataclass schema first and wrapping ``__init__`` preserves + domain ``ConversionError`` contracts while keeping standard Pydantic + ``ValidationError`` behavior for type and constraint failures. + + :param cls: Pydantic dataclass to patch. + :type cls: type[ModelT] + :return: ``cls`` with its ``__init__`` patched. + :rtype: type[ModelT] + """ + rebuild_dataclass(cls) + generated_init = cls.__init__ + + def _init(self: ModelT, *args: object, **kwargs: object) -> None: + try: + generated_init(self, *args, **kwargs) + except ValidationError as error: + causes = [item.get("ctx", {}).get("error") for item in error.errors()] + if causes and all(isinstance(cause, ConversionError) for cause in causes): + raise causes[0] from None + raise + + cls.__init__ = _init # pyright: ignore[reportAttributeAccessIssue] + return cls diff --git a/src/cdm_data_loaders/converters/core/extensions.py b/src/cdm_data_loaders/converters/core/extensions.py index 75a72fcc..6ffee97e 100644 --- a/src/cdm_data_loaders/converters/core/extensions.py +++ b/src/cdm_data_loaders/converters/core/extensions.py @@ -13,6 +13,11 @@ from cdm_data_loaders.converters.core.errors import ConversionError from cdm_data_loaders.converters.core.ir_values import Value, freeze_mapping, mutable_value from cdm_data_loaders.validation.jsonschema.metaschema import get_xsv_config_schema +from typing import Annotated +from pydantic import AfterValidator, ConfigDict, InstanceOf, WithJsonSchema +from pydantic.dataclasses import dataclass as pydantic_dataclass +from cdm_data_loaders.converters.core.errors import ConversionError, unwrap_conversion_errors + type ExtensionSchemaLoader = Callable[[], Mapping[str, object] | bool] @@ -127,21 +132,33 @@ def __hash__(self) -> int: return hash((self.namespace, self.schema)) -@dataclass(frozen=True, slots=True) +def _unique_extension_specs(specs: tuple[ExtensionSpec, ...]) -> tuple[ExtensionSpec, ...]: + """Reject duplicate extension namespaces within a registry declaration. + + :param specs: Validated tuple of ``ExtensionSpec`` entries. + :type specs: tuple[ExtensionSpec, ...] + :return: Validated tuple of ``ExtensionSpec`` entries. + :rtype: tuple[ExtensionSpec, ...] + :raises ExtensionError: If two specs declare the same namespace. + """ + if len({spec.namespace for spec in specs}) != len(specs): + msg = "Duplicate extension namespace" + raise ExtensionError(msg) + return specs + + +type ExtensionSpecEntry = Annotated[ + InstanceOf[ExtensionSpec], + WithJsonSchema({"type": "object", "title": "ExtensionSpec"}), +] + + +@unwrap_conversion_errors +@pydantic_dataclass(frozen=True, slots=True, config=ConfigDict(extra="forbid")) class ExtensionRegistry: """A closed set of namespace contracts; additions produce a new registry.""" - specs: tuple[ExtensionSpec, ...] = () - - def __post_init__(self) -> None: - """Reject duplicate contracts and isolate the declaration sequence.""" - object.__setattr__(self, "specs", tuple(self.specs)) - if any(not isinstance(spec, ExtensionSpec) for spec in self.specs): - msg = "Registry entries must be ExtensionSpec instances" - raise TypeError(msg) - if len({spec.namespace for spec in self.specs}) != len(self.specs): - msg = "Duplicate extension namespace" - raise ExtensionError(msg) + specs: Annotated[tuple[ExtensionSpecEntry, ...], AfterValidator(_unique_extension_specs)] = () def register(self, spec: ExtensionSpec, *, replace: bool = False) -> "ExtensionRegistry": """Add a contract, requiring an explicit flag to replace an existing one.""" diff --git a/src/cdm_data_loaders/converters/core/ir.py b/src/cdm_data_loaders/converters/core/ir.py index e574ab71..4ebb62dc 100644 --- a/src/cdm_data_loaders/converters/core/ir.py +++ b/src/cdm_data_loaders/converters/core/ir.py @@ -1,65 +1,158 @@ """Immutable, target-independent schema facts and composition contracts.""" -from collections.abc import Mapping -from dataclasses import dataclass, field -from typing import Final, Literal, Protocol +from collections.abc import Callable, Mapping +from dataclasses import field +from typing import Annotated, Final, Literal, Protocol from frozendict import frozendict +from pydantic import ( + AfterValidator, + ConfigDict, + GetPydanticSchema, + InstanceOf, + StrictBool, + StrictInt, + StrictStr, +) +from pydantic import Field as PydanticField +from pydantic.dataclasses import dataclass as pydantic_dataclass +from pydantic_core import core_schema + +from cdm_data_loaders.converters.core.errors import ConversionError, unwrap_conversion_errors +from cdm_data_loaders.converters.core.extensions import Extensions, validated_extensions +from cdm_data_loaders.converters.core.ir_values import Value, freeze_dict, freeze_mapping -from cdm_data_loaders.converters.core.errors import ConversionError -from cdm_data_loaders.converters.core.extensions import validated_extensions -from cdm_data_loaders.converters.core.ir_values import Value, freeze_mapping +IR_DATACLASS_CONFIG: Final[ConfigDict] = ConfigDict(extra="forbid") type NodeType = Literal[ "object", "map", "array", "string", "integer", "number", "boolean", "null", "any", "never", "unknown" ] type Source = Literal["json-schema", "dlt", "iceberg", "pydantic", "xsd", "pyarrow"] +type NonNegativeStrictInt = Annotated[StrictInt, PydanticField(ge=0)] + + +def _validate_frozen_mapping(values: Mapping[str, Value]) -> Mapping[str, Value]: + """Validate and deep-freeze a JSON-shaped metadata mapping. + + :param values: Candidate metadata mapping. + :type values: Mapping[str, Value] + :return: Deep-frozen schema mapping. + :rtype: Mapping[str, Value] + :raises ValueError: If ``values`` contains non-string keys or unsupported values. + """ + try: + return freeze_mapping(values) + except TypeError as error: + raise ValueError(str(error)) from error + + +def _schema_metadata(values: Mapping[str, Value]) -> Mapping[str, Value]: + """Own schema metadata without accepting alternate extension declarations. + + :param values: Candidate schema metadata mapping. + :type values: Mapping[str, Value] + :return: Deep-frozen schema metadata mapping. + :rtype: Mapping[str, Value] + :raises ConversionError: If a top-level key starts with ``x-``. + """ + owned = _validate_frozen_mapping(values) + for name in owned: + if name.startswith("x-"): + msg = f"Extension namespace {name} must use extensions" + raise ConversionError(msg) + return owned + + +def _validate_schema_metadata(values: Mapping[str, Value]) -> Mapping[str, Value]: + """Validate and freeze schema constraints or annotations via ``_schema_metadata``. + + :param values: Candidate schema metadata mapping. + :type values: Mapping[str, Value] + :return: Deep-frozen schema metadata mapping. + :rtype: Mapping[str, Value] + :raises ValueError: If ``values`` contains invalid keys or values. + :raises ConversionError: If a top-level key starts with ``x-``. + """ + return _schema_metadata(values) + + +def _validate_extensions_map(values: Mapping[str, Value]) -> Extensions: + """Validate an extension mapping or preserve an already-validated ``Extensions`` object. + + :param values: Candidate extension mapping or ``Extensions`` instance. + :type values: Mapping[str, Value] + :return: Validated immutable ``Extensions`` mapping. + :rtype: Extensions + :raises ValueError: If ``values`` contains non-string keys or unsupported values. + :raises ExtensionError: If any namespace or payload violates its contract. + """ + try: + return validated_extensions(values) + except TypeError as error: + raise ValueError(str(error)) from error + + +def _mapping_validator_schema( + validator: Callable[[Mapping[str, Value]], Mapping[str, Value]], +) -> GetPydanticSchema: + """Build a Pydantic schema checking ``Mapping`` first and then applying ``validator``. + + :param validator: Mapping validator returning an immutable mapping. + :type validator: Callable[[Mapping[str, Value]], Mapping[str, Value]] + :return: ``GetPydanticSchema`` annotation metadata. + :rtype: GetPydanticSchema + """ + return GetPydanticSchema( + lambda _tp, _handler: core_schema.chain_schema( + [ + core_schema.is_instance_schema(Mapping), + core_schema.no_info_plain_validator_function(validator), + ] + ) + ) + + +def _unique_fields(fields: tuple["Field", ...]) -> tuple["Field", ...]: + """Reject duplicate property names in an object node. + + :param fields: Validated property ``Field`` tuple. + :type fields: tuple[Field, ...] + :return: Validated property ``Field`` tuple. + :rtype: tuple[Field, ...] + :raises ValueError: If two ``Field`` entries share the same ``name``. + """ + if len({prop.name for prop in fields}) != len(fields): + msg = "Duplicate property names" + raise ValueError(msg) + return fields + + +type FrozenValueMap = Annotated[Mapping[str, Value], _mapping_validator_schema(_validate_frozen_mapping)] +type SchemaMetadataMap = Annotated[Mapping[str, Value], _mapping_validator_schema(_validate_schema_metadata)] +type ExtensionsMap = Annotated[Mapping[str, Value], _mapping_validator_schema(_validate_extensions_map)] +type NodeMap = Annotated[Mapping[StrictStr, InstanceOf["TypedNode"]], AfterValidator(freeze_dict)] +type NestedNodeMap = Annotated[Mapping[StrictStr, NodeMap], AfterValidator(freeze_dict)] -@dataclass(frozen=True, slots=True) +@pydantic_dataclass(frozen=True, slots=True, config=IR_DATACLASS_CONFIG) class Provenance: """Source location and owned metadata, not an alternate conversion input.""" source: Source - path: tuple[str | int, ...] = () - metadata: Mapping[str, Value] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - - def __post_init__(self) -> None: - """Isolate source paths and metadata.""" - if self.source not in {"json-schema", "dlt", "iceberg", "pydantic", "xsd", "pyarrow"}: - msg = f"Unknown schema source: {self.source}" - raise ValueError(msg) - if any(not isinstance(part, (str, int)) or isinstance(part, bool) for part in self.path): - msg = "Source paths require string or integer segments" - raise TypeError(msg) - object.__setattr__(self, "path", tuple(self.path)) - object.__setattr__(self, "metadata", freeze_mapping(self.metadata)) - - -@dataclass(frozen=True, slots=True) + path: tuple[StrictStr | StrictInt, ...] = () + metadata: FrozenValueMap = field(default_factory=frozendict) + + +@pydantic_dataclass(frozen=True, slots=True, config=IR_DATACLASS_CONFIG) class NodeHints: """Source logical facts; target encodings and fallbacks belong to emitters.""" - logical_type: str | None = None - precision: int | None = None - scale: int | None = None - bit_width: int | None = None - timezone: bool | None = None - length: int | None = None - - def __post_init__(self) -> None: - """Reject malformed logical type facts without coercion.""" - if self.logical_type is not None and not isinstance(self.logical_type, str): - msg = "logical_type must be a string" - raise TypeError(msg) - for name in ("precision", "scale", "bit_width", "length"): - value = getattr(self, name) - if value is not None and (type(value) is not int or value < 0): - msg = f"{name} must be a nonnegative integer" - raise ValueError(msg) - if self.timezone is not None and type(self.timezone) is not bool: - msg = "timezone must be a boolean or None" - raise TypeError(msg) + logical_type: StrictStr | None = None + precision: NonNegativeStrictInt | None = None + scale: NonNegativeStrictInt | None = None + bit_width: NonNegativeStrictInt | None = None + timezone: StrictBool | None = None + length: NonNegativeStrictInt | None = None SPEC_BOOL: Final[tuple[NodeType, NodeHints]] = ("boolean", NodeHints(logical_type="boolean")) @@ -86,7 +179,7 @@ def __post_init__(self) -> None: SPEC_STRING: Final[tuple[NodeType, NodeHints]] = ("string", NodeHints(logical_type="string")) -@dataclass(frozen=True, slots=True) +@pydantic_dataclass(frozen=True, slots=True, config=IR_DATACLASS_CONFIG) class TypedNode: """A value schema, independent of a property's presence requirement. @@ -96,101 +189,36 @@ class TypedNode: """ type: NodeType - nullable: bool | None = None - declared_type: str | tuple[str, ...] | None = None - inferred_type: str | None = None - hints: NodeHints = field(default_factory=NodeHints) - properties: tuple["Field", ...] = () - required_names: tuple[str, ...] | None = None - items: "TypedNode | tuple[TypedNode, ...] | None" = None - prefix_items: tuple["TypedNode", ...] | None = None - pattern_properties: Mapping[str, "TypedNode"] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] # pyright: ignore[reportAssignmentType] - additional_properties: "TypedNode | None" = None - any_of: tuple["TypedNode", ...] | None = None - one_of: tuple["TypedNode", ...] | None = None - schema_keywords: Mapping[str, "TypedNode"] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - schema_maps: Mapping[str, Mapping[str, "TypedNode"]] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - key_type: "TypedNode | None" = None - value_type: "TypedNode | None" = None - constraints: Mapping[str, Value] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - annotations: Mapping[str, Value] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - extensions: Mapping[str, Value] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - provenance: Provenance | None = None - source_keywords: tuple[str, ...] = () - - def __post_init__(self) -> None: - """Own mappings and sequences and validate child structure.""" - self._validate_types() - object.__setattr__(self, "source_keywords", _string_tuple(self.source_keywords)) - if self.declared_type is not None and not isinstance(self.declared_type, str): - object.__setattr__(self, "declared_type", _string_tuple(self.declared_type)) - object.__setattr__(self, "properties", tuple(self.properties)) - if any(not isinstance(prop, Field) for prop in self.properties): - msg = "properties must contain Field instances" - raise TypeError(msg) - if len({prop.name for prop in self.properties}) != len(self.properties): - msg = "Duplicate property names" - raise ValueError(msg) - if self.required_names is not None: - object.__setattr__(self, "required_names", _string_tuple(self.required_names)) - self._own_children() - object.__setattr__(self, "constraints", _schema_metadata(self.constraints)) - _init_common_metadata(self) - - def _validate_types(self) -> None: - """Reject invalid scalar facts and mutable stand-ins for typed models.""" - if self.type not in { - "object", - "map", - "array", - "string", - "integer", - "number", - "boolean", - "null", - "any", - "never", - "unknown", - }: - msg = f"Unknown node type: {self.type}" - raise ValueError(msg) - if self.nullable is not None and type(self.nullable) is not bool: - msg = "nullable must be a boolean or None" - raise TypeError(msg) - if self.inferred_type is not None and not isinstance(self.inferred_type, str): - msg = "inferred_type must be a string or None" - raise TypeError(msg) - if not isinstance(self.hints, NodeHints): - msg = "hints must be NodeHints" - raise TypeError(msg) - - def _own_children(self) -> None: - """Copy each schema-valued child collection into immutable containers.""" - for name in ("prefix_items", "any_of", "one_of"): - sequence = getattr(self, name) - if sequence is not None: - object.__setattr__(self, name, _node_tuple(sequence)) - if isinstance(self.items, (list, tuple)): - object.__setattr__(self, "items", _node_tuple(self.items)) - elif self.items is not None and not isinstance(self.items, TypedNode): - msg = "items must be a node or tuple of nodes" - raise TypeError(msg) - for name in ("additional_properties", "key_type", "value_type"): - child = getattr(self, name) - if child is not None and not isinstance(child, TypedNode): - msg = f"{name} must be a node" - raise TypeError(msg) - for name in ("pattern_properties", "schema_keywords"): - object.__setattr__(self, name, _node_mapping(getattr(self, name))) - _string_tuple(tuple(self.schema_maps)) - object.__setattr__( - self, - "schema_maps", - frozendict({name: _node_mapping(children) for name, children in self.schema_maps.items()}), - ) + nullable: StrictBool | None = None + declared_type: StrictStr | tuple[StrictStr, ...] | None = None + inferred_type: StrictStr | None = None + hints: InstanceOf[NodeHints] = field(default_factory=NodeHints) + properties: Annotated[tuple[InstanceOf["Field"], ...], AfterValidator(_unique_fields)] = () + required_names: tuple[StrictStr, ...] | None = None + items: InstanceOf["TypedNode"] | tuple[InstanceOf["TypedNode"], ...] | None = None + prefix_items: tuple[InstanceOf["TypedNode"], ...] | None = None + pattern_properties: NodeMap = field(default_factory=frozendict) + additional_properties: InstanceOf["TypedNode"] | None = None + any_of: tuple[InstanceOf["TypedNode"], ...] | None = None + one_of: tuple[InstanceOf["TypedNode"], ...] | None = None + schema_keywords: NodeMap = field(default_factory=frozendict) + schema_maps: NestedNodeMap = field(default_factory=frozendict) + key_type: InstanceOf["TypedNode"] | None = None + value_type: InstanceOf["TypedNode"] | None = None + constraints: SchemaMetadataMap = field(default_factory=frozendict) + annotations: SchemaMetadataMap = field(default_factory=frozendict) + extensions: ExtensionsMap = field(default_factory=Extensions) + provenance: InstanceOf[Provenance] | None = None + source_keywords: tuple[StrictStr, ...] = () def aggregate_metadata(self, field: "Field | None" = None) -> Mapping[str, Value]: - """Merge constraints, annotations, and extensions into a single mapping.""" + """Merge constraints, annotations, and extensions into a single mapping. + + :param field: Optional enclosing property ``Field`` whose metadata overrides node metadata. + :type field: Field | None + :return: Merged metadata mapping. + :rtype: Mapping[str, Value] + """ all_meta = {**self.constraints, **self.annotations, **self.extensions} if field is not None: all_meta.update(field.annotations) @@ -198,100 +226,30 @@ def aggregate_metadata(self, field: "Field | None" = None) -> Mapping[str, Value return all_meta -def _schema_metadata(values: Mapping[str, Value]) -> Mapping[str, Value]: - """Own schema metadata without accepting alternate extension declarations.""" - owned = freeze_mapping(values) - for name in owned: - if name.startswith("x-"): - msg = f"Extension namespace {name} must use extensions" - raise ConversionError(msg) - return owned - - -def _string_tuple(values: tuple[str, ...]) -> tuple[str, ...]: - """Own ordered string declarations without accepting a scalar string.""" - if not isinstance(values, (list, tuple)) or any(not isinstance(value, str) for value in values): - msg = "Expected a list or tuple of strings" - raise TypeError(msg) - return tuple(values) - - -def _node_tuple(values: tuple[TypedNode, ...]) -> tuple[TypedNode, ...]: - """Own a sequence of immutable schema nodes.""" - if any(not isinstance(value, TypedNode) for value in values): - msg = "Expected TypedNode values" - raise TypeError(msg) - return tuple(values) - - -def _validate_provenance(value: Provenance | None) -> None: - """Reject mutable or untyped provenance substitutes.""" - if value is not None and not isinstance(value, Provenance): - msg = "provenance must be Provenance or None" - raise TypeError(msg) - - -def _init_common_metadata(obj: "TypedNode | Field | SchemaDocument") -> None: - """Validate provenance and freeze annotations and extensions on an IR model. - - :param obj: IR node, field, or document instance being initialized. - :type obj: TypedNode | Field | SchemaDocument - """ - _validate_provenance(obj.provenance) - object.__setattr__(obj, "annotations", _schema_metadata(obj.annotations)) - object.__setattr__(obj, "extensions", validated_extensions(obj.extensions)) - - -def _node_mapping(nodes: Mapping[str, TypedNode]) -> Mapping[str, TypedNode]: - """Own a named collection of immutable nodes.""" - if any(not isinstance(key, str) or not isinstance(value, TypedNode) for key, value in nodes.items()): - msg = "Expected string keys and TypedNode values" - raise TypeError(msg) - return frozendict(nodes) - - -@dataclass(frozen=True, slots=True) +@pydantic_dataclass(frozen=True, slots=True, config=IR_DATACLASS_CONFIG) class Field: """Property presence and metadata outside any nullable value wrapper.""" - name: str - node: TypedNode - required: bool = False - annotations: Mapping[str, Value] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - extensions: Mapping[str, Value] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - provenance: Provenance | None = None + name: StrictStr + node: InstanceOf[TypedNode] + required: StrictBool = False + annotations: SchemaMetadataMap = field(default_factory=frozendict) + extensions: ExtensionsMap = field(default_factory=Extensions) + provenance: InstanceOf[Provenance] | None = None - def __post_init__(self) -> None: - """Validate field identity and own its metadata.""" - if not isinstance(self.name, str) or not isinstance(self.node, TypedNode) or type(self.required) is not bool: - msg = "Fields require a string name, node and boolean required flag" - raise TypeError(msg) - _init_common_metadata(self) - -@dataclass(frozen=True, slots=True) +@pydantic_dataclass(frozen=True, slots=True, config=IR_DATACLASS_CONFIG) class SchemaDocument: """A named schema root with document metadata separate from value metadata.""" - root: TypedNode - name: str | None = None - dialect: str | None = None - identifier: str | None = None - annotations: Mapping[str, Value] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - extensions: Mapping[str, Value] = field(default_factory=frozendict) # pyright: ignore[reportAssignmentType] - provenance: Provenance | None = None - jsonschema_validator_cls: type | None = None - - def __post_init__(self) -> None: - """Validate the root and isolate document metadata.""" - if not isinstance(self.root, TypedNode): - msg = "A schema document requires a TypedNode root" - raise TypeError(msg) - for name in ("name", "dialect", "identifier"): - if getattr(self, name) is not None and not isinstance(getattr(self, name), str): - msg = f"{name} must be a string or None" - raise TypeError(msg) - _init_common_metadata(self) + root: InstanceOf[TypedNode] + name: StrictStr | None = None + dialect: StrictStr | None = None + identifier: StrictStr | None = None + annotations: SchemaMetadataMap = field(default_factory=frozendict) + extensions: ExtensionsMap = field(default_factory=Extensions) + provenance: InstanceOf[Provenance] | None = None + jsonschema_validator_cls: InstanceOf[type] | None = None class Emitter[Output](Protocol): @@ -300,3 +258,7 @@ class Emitter[Output](Protocol): def emit(self, document: SchemaDocument) -> Output: """Produce a target schema from the document tree.""" ... + + +for _ir_cls in (TypedNode, Field, SchemaDocument): + unwrap_conversion_errors(_ir_cls) diff --git a/src/cdm_data_loaders/converters/core/ir_values.py b/src/cdm_data_loaders/converters/core/ir_values.py index eb6824dd..f6b94807 100644 --- a/src/cdm_data_loaders/converters/core/ir_values.py +++ b/src/cdm_data_loaders/converters/core/ir_values.py @@ -79,3 +79,14 @@ def mutable_value(value: Value) -> MutableValue: if isinstance(value, tuple): return [mutable_value(item) for item in value] return value + + +def freeze_dict[K, V](mapping: Mapping[K, V]) -> frozendict[K, V]: + """Copy a validated mapping into a shallow ``frozendict``. + + :param mapping: Validated mapping to freeze. + :type mapping: Mapping[K, V] + :return: Shallow ``frozendict`` copy. + :rtype: frozendict[K, V] + """ + return frozendict(mapping) diff --git a/src/cdm_data_loaders/converters/core/metadata.py b/src/cdm_data_loaders/converters/core/metadata.py index 27deac9b..70f07c66 100644 --- a/src/cdm_data_loaders/converters/core/metadata.py +++ b/src/cdm_data_loaders/converters/core/metadata.py @@ -1,10 +1,10 @@ """Draft-specific JSON Schema keyword selection for field metadata.""" -from dataclasses import dataclass from functools import cache from typing import Final from jsonschema import Draft7Validator +from pydantic import BaseModel, ConfigDict, Field from cdm_data_loaders.validation.jsonschema.keywords import ( REF_AND_IDENTITY_KEYWORDS, @@ -26,12 +26,13 @@ def metadata_keys_for(validator_cls: type) -> frozenset[str]: return frozenset(get_known_jsonschema_keywords(validator_cls) - STRUCTURAL_OR_COMPOSITIONAL_KEYWORDS) -@dataclass(frozen=True) -class ConversionContext: +class ConversionContext(BaseModel): """Draft-specific metadata selection for a single emission.""" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) + validator_cls: type - extra_metadata_keywords: frozenset[str] = frozenset() + extra_metadata_keywords: frozenset[str] = Field(default_factory=frozenset) @property def metadata_keys(self) -> frozenset[str]: diff --git a/src/cdm_data_loaders/converters/emitters/_tabular_base.py b/src/cdm_data_loaders/converters/emitters/_tabular_base.py index fd471270..27cd6c4e 100644 --- a/src/cdm_data_loaders/converters/emitters/_tabular_base.py +++ b/src/cdm_data_loaders/converters/emitters/_tabular_base.py @@ -119,7 +119,10 @@ def _logger(self) -> logging.Logger: def build_context(self, validator_cls: type) -> ConversionContext: """Build draft-specific metadata policy and report invalid requested keywords.""" - context = ConversionContext(validator_cls, self.extra_metadata_keywords) + context = ConversionContext( + validator_cls=validator_cls, + extra_metadata_keywords=self.extra_metadata_keywords, + ) if context.invalid_extra_metadata_keywords: self._logger().warning( "Ignoring invalid extra_metadata_keywords %s: these are neither recognised JSON Schema " diff --git a/src/cdm_data_loaders/converters/emitters/dlt.py b/src/cdm_data_loaders/converters/emitters/dlt.py index 012c1120..84117b87 100644 --- a/src/cdm_data_loaders/converters/emitters/dlt.py +++ b/src/cdm_data_loaders/converters/emitters/dlt.py @@ -272,7 +272,9 @@ def _table( table["columns"] = columns if "description" in node.annotations: table["description"] = cast("str", mutable_value(node.annotations["description"])) - tree = DltTableNode(name, parent, child_key(parent, name) if parent else name, table, children) + tree = DltTableNode( + name=name, parent=parent, key=child_key(parent, name) if parent else name, table=table, children=children + ) if native: tree = _restore_table(tree, node) if parent is None and self.write_disposition is not None: @@ -310,11 +312,11 @@ def _child(self, prop: SchemaField, parent: str, depth: int, *, native: bool = F value = ( new_column("value", _scalar_type(item)) if _is_json(node) - else self._column(SchemaField("value", item), native=native) + else self._column(SchemaField(name="value", node=item), native=native) ) table = new_table(name, parent_table_name=parent) table["columns"] = {"value": value} - child = DltTableNode(name, parent, child_key(parent, name), table) + child = DltTableNode(name=name, parent=parent, key=child_key(parent, name), table=table) if child is not None: if native: child = _restore_table(child, node) diff --git a/src/cdm_data_loaders/converters/emitters/json_schema/emitter.py b/src/cdm_data_loaders/converters/emitters/json_schema/emitter.py index c07acebe..c0092924 100644 --- a/src/cdm_data_loaders/converters/emitters/json_schema/emitter.py +++ b/src/cdm_data_loaders/converters/emitters/json_schema/emitter.py @@ -1,8 +1,9 @@ """Emit JSON Schema from typed schema facts and scoped metadata.""" -from dataclasses import dataclass from typing import Any +from pydantic import BaseModel, ConfigDict, StrictBool + from cdm_data_loaders.converters.core.errors import ConversionError from cdm_data_loaders.converters.core.ir import Field, SchemaDocument, TypedNode from cdm_data_loaders.converters.core.ir_values import mutable_value @@ -21,20 +22,12 @@ ) -@dataclass(frozen=True, slots=True) -class JsonSchemaEmitter: +class JsonSchemaEmitter(BaseModel): """Render typed IR trees using each source's JSON Schema compatibility policy.""" - preserve_unknown_hints: bool = True - - def __post_init__(self) -> None: - """Reject non-boolean hint policies. + model_config = ConfigDict(extra="forbid", frozen=True) - :raises TypeError: If ``preserve_unknown_hints`` is not a ``bool``. - """ - if type(self.preserve_unknown_hints) is not bool: - msg = "preserve_unknown_hints must be a boolean" - raise TypeError(msg) + preserve_unknown_hints: StrictBool = True def emit(self, document: SchemaDocument) -> dict[str, Any]: """Return a fresh JSON Schema document dictionary for ``document``. diff --git a/src/cdm_data_loaders/converters/emitters/linkml.py b/src/cdm_data_loaders/converters/emitters/linkml.py index 63f8ac57..266bd8b4 100644 --- a/src/cdm_data_loaders/converters/emitters/linkml.py +++ b/src/cdm_data_loaders/converters/emitters/linkml.py @@ -1,7 +1,6 @@ """Emit official LinkML schema models from typed schema documents.""" import re -from dataclasses import dataclass, field from typing import Final, cast from linkml_runtime.linkml_model.meta import ( @@ -12,6 +11,7 @@ SchemaDefinition, SlotDefinition, ) +from pydantic import BaseModel, ConfigDict, PrivateAttr from cdm_data_loaders.converters.core.ir import Field as SchemaField from cdm_data_loaders.converters.core.ir import SchemaDocument, TypedNode @@ -41,14 +41,15 @@ UNSIGNED_LOGICAL_INTS: Final[frozenset[str]] = frozenset({"uint8", "uint16", "uint32", "uint64"}) -@dataclass(slots=True) -class LinkMLEmitter: +class LinkMLEmitter(BaseModel): """Render a document as a LinkML SchemaDefinition.""" + model_config = ConfigDict(extra="forbid", frozen=True) + schema_name: str | None = None - _classes: dict[str, ClassDefinition] = field(init=False, default_factory=dict) - _enums: dict[str, EnumDefinition] = field(init=False, default_factory=dict) - _used_names: set[str] = field(init=False, default_factory=set) + _classes: dict[str, ClassDefinition] = PrivateAttr(default_factory=dict) + _enums: dict[str, EnumDefinition] = PrivateAttr(default_factory=dict) + _used_names: set[str] = PrivateAttr(default_factory=set) def emit(self, document: SchemaDocument) -> SchemaDefinition: """Emit a standalone LinkML schema model. @@ -210,8 +211,8 @@ def _node_expression(self, node: TypedNode, owner_name: str, field_name: str) -> type="object", nullable=False, properties=( - SchemaField("key", node.key_type, required=True), - SchemaField("value", node.value_type, required=value_required), + SchemaField(name="key", node=node.key_type, required=True), + SchemaField(name="value", node=node.value_type, required=value_required), ), provenance=node.provenance, ) diff --git a/src/cdm_data_loaders/converters/facades/_base.py b/src/cdm_data_loaders/converters/facades/_base.py index 1741b4ba..4c5ea860 100644 --- a/src/cdm_data_loaders/converters/facades/_base.py +++ b/src/cdm_data_loaders/converters/facades/_base.py @@ -49,7 +49,7 @@ def _reader(self) -> JsonSchemaIRReader: :rtype: JsonSchemaIRReader """ return JsonSchemaReader( - self.extension_registry, + extension_registry=self.extension_registry, error_type=self._error_cls, converter_name=self._converter_name or type(self).__name__, dereference_function=self._dereference_function, diff --git a/src/cdm_data_loaders/converters/facades/jsonschema_to_pyarrow.py b/src/cdm_data_loaders/converters/facades/jsonschema_to_pyarrow.py index 8f0989b2..40233f5d 100644 --- a/src/cdm_data_loaders/converters/facades/jsonschema_to_pyarrow.py +++ b/src/cdm_data_loaders/converters/facades/jsonschema_to_pyarrow.py @@ -73,7 +73,7 @@ def _reader(self) -> StrictJsonSchemaReader: """ forbidden_formats = frozenset(fmt for fmt, dtype in self.format_map.items() if is_non_json_arrow_type(dtype)) return StrictJsonSchemaReader( - self.extension_registry, + extension_registry=self.extension_registry, error_type=self._error_cls, converter_name=self._converter_name, max_recursion_depth=self.max_recursion_depth, diff --git a/src/cdm_data_loaders/converters/facades/pyiceberg_to_jsonschema.py b/src/cdm_data_loaders/converters/facades/pyiceberg_to_jsonschema.py index 7fb43044..044f583a 100644 --- a/src/cdm_data_loaders/converters/facades/pyiceberg_to_jsonschema.py +++ b/src/cdm_data_loaders/converters/facades/pyiceberg_to_jsonschema.py @@ -17,7 +17,7 @@ def convert_type( return cast( "dict[str, Any]", JsonSchemaEmitter().emit_node( - IcebergReader(extension_registry, mapping_target="JSON Schema").read_node(field_type) + IcebergReader(extension_registry=extension_registry, mapping_target="JSON Schema").read_node(field_type) ), ) @@ -27,7 +27,7 @@ def convert_field(field: NestedField, *, extension_registry: ExtensionRegistry = return cast( "dict[str, Any]", JsonSchemaEmitter().emit_field( - IcebergReader(extension_registry, mapping_target="JSON Schema").read_field(field) + IcebergReader(extension_registry=extension_registry, mapping_target="JSON Schema").read_field(field) ), ) @@ -58,6 +58,8 @@ def table_to_json_schema( :type schema: dict[str, Any] | None :returns: JSON Schema document, or the destination namespace document when ``schema`` is given. """ - document = IcebergReader(extension_registry, mapping_target="JSON Schema").read_table(table, identifier) + document = IcebergReader(extension_registry=extension_registry, mapping_target="JSON Schema").read_table( + table, identifier + ) emitter = JsonSchemaEmitter() return emitter.emit(document) if schema is None else emitter.emit_into(document, schema) diff --git a/src/cdm_data_loaders/converters/facades/pyiceberg_to_linkml.py b/src/cdm_data_loaders/converters/facades/pyiceberg_to_linkml.py index 68ca48f3..bbb8810e 100644 --- a/src/cdm_data_loaders/converters/facades/pyiceberg_to_linkml.py +++ b/src/cdm_data_loaders/converters/facades/pyiceberg_to_linkml.py @@ -20,7 +20,9 @@ def convert_type( :type extension_registry: ExtensionRegistry :returns: LinkML slot definition. """ - return LinkMLEmitter().emit_node(IcebergReader(extension_registry, mapping_target="LinkML").read_node(field_type)) + return LinkMLEmitter().emit_node( + IcebergReader(extension_registry=extension_registry, mapping_target="LinkML").read_node(field_type) + ) def convert_field(field: NestedField, *, extension_registry: ExtensionRegistry = DEFAULT_EXTENSIONS) -> SlotDefinition: @@ -32,7 +34,9 @@ def convert_field(field: NestedField, *, extension_registry: ExtensionRegistry = :type extension_registry: ExtensionRegistry :returns: Named LinkML slot definition. """ - return LinkMLEmitter().emit_field(IcebergReader(extension_registry, mapping_target="LinkML").read_field(field)) + return LinkMLEmitter().emit_field( + IcebergReader(extension_registry=extension_registry, mapping_target="LinkML").read_field(field) + ) def convert_struct( @@ -68,6 +72,8 @@ def table_to_linkml( :type schema: SchemaDefinition | None :returns: Schema model suitable for LinkML's YAML dumper. """ - document = IcebergReader(extension_registry, mapping_target="LinkML").read_table(table, identifier) + document = IcebergReader(extension_registry=extension_registry, mapping_target="LinkML").read_table( + table, identifier + ) emitter = LinkMLEmitter() return emitter.emit(document) if schema is None else emitter.emit_into(document, schema) diff --git a/src/cdm_data_loaders/converters/sources/dlt.py b/src/cdm_data_loaders/converters/sources/dlt.py index c260cc29..41744b21 100644 --- a/src/cdm_data_loaders/converters/sources/dlt.py +++ b/src/cdm_data_loaders/converters/sources/dlt.py @@ -2,12 +2,15 @@ import logging from collections.abc import Mapping -from dataclasses import dataclass, replace +from dataclasses import replace from typing import Any, Final, Literal +from pydantic import BaseModel, ConfigDict, InstanceOf, StrictBool +from pydantic import Field as PydanticField + from cdm_data_loaders.converters.core.dlt_normalization import DltTableNode, unflatten_tables from cdm_data_loaders.converters.core.errors import ConversionError -from cdm_data_loaders.converters.core.extensions import DEFAULT_EXTENSIONS, ExtensionRegistry, Extensions +from cdm_data_loaders.converters.core.extensions import ExtensionRegistry, Extensions, default_extension_registry from cdm_data_loaders.converters.core.ir import Field, NodeHints, NodeType, Provenance, SchemaDocument, TypedNode logger = logging.getLogger(__name__) @@ -27,23 +30,15 @@ } -@dataclass(frozen=True, slots=True) -class DltReader: +class DltReader(BaseModel): """Resolve filtering and ambiguous child-table shape at the reader boundary.""" - include_dlt_columns: bool = False - include_variant_columns: bool = False - child_table_mode: Literal["object", "array"] = "object" - extension_registry: ExtensionRegistry = DEFAULT_EXTENSIONS + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) - def __post_init__(self) -> None: - """Reject invalid policy configuration.""" - if self.child_table_mode not in {"object", "array"}: - msg = "child_table_mode must be 'object' or 'array'" - raise ValueError(msg) - if type(self.include_dlt_columns) is not bool or type(self.include_variant_columns) is not bool: - msg = "Column filters must be booleans" - raise TypeError(msg) + include_dlt_columns: StrictBool = False + include_variant_columns: StrictBool = False + child_table_mode: Literal["object", "array"] = "object" + extension_registry: InstanceOf[ExtensionRegistry] = PydanticField(default_factory=default_extension_registry) def read(self, source: Mapping[str, Any]) -> dict[str, SchemaDocument]: """Return ordered root documents from stored-schema or plain-table input.""" @@ -111,7 +106,7 @@ def _table(self, table: DltTableNode) -> TypedNode: if required: required_names.append(name) annotations = {"description": table.table["description"]} if "description" in table.table else {} - provenance = Provenance("dlt", ("tables", table.name), table.table) + provenance = Provenance(source="dlt", path=("tables", table.name), metadata=table.table) object_node = TypedNode( type="object", nullable=False, @@ -140,7 +135,7 @@ def _column(self, name: str, column: Mapping[str, Any], table_name: str) -> Fiel kind = TYPES.get(data_type, "any" if data_type is None else "unknown") if kind == "unknown": logger.warning("Unknown dlt data_type %r at %s.%s retained in IR", data_type, table_name, name) - provenance = Provenance("dlt", ("tables", table_name, "columns", name), column) + provenance = Provenance(source="dlt", path=("tables", table_name, "columns", name), metadata=column) nullable = column.get("nullable", True) hints = NodeHints( logical_type=data_type, @@ -149,7 +144,7 @@ def _column(self, name: str, column: Mapping[str, Any], table_name: str) -> Fiel bit_width=column.get("precision") if data_type == "bigint" else None, timezone=column.get("timezone"), ) - extensions = Extensions({"x-dlt": dict(column)}, self.extension_registry) + extensions = Extensions(payload={"x-dlt": dict(column)}, registry=self.extension_registry) node = TypedNode( type=kind, nullable=nullable, diff --git a/src/cdm_data_loaders/converters/sources/iceberg.py b/src/cdm_data_loaders/converters/sources/iceberg.py index b3059b80..442c1b43 100644 --- a/src/cdm_data_loaders/converters/sources/iceberg.py +++ b/src/cdm_data_loaders/converters/sources/iceberg.py @@ -2,11 +2,13 @@ from base64 import b64encode from collections.abc import Mapping -from dataclasses import dataclass, replace +from dataclasses import replace from datetime import date, datetime, time from typing import Final from uuid import UUID +from pydantic import BaseModel, ConfigDict, InstanceOf +from pydantic import Field as PydanticField from pyiceberg.schema import Schema from pyiceberg.table import Table from pyiceberg.types import ( @@ -32,7 +34,7 @@ ) from cdm_data_loaders.converters.core.errors import ConversionError -from cdm_data_loaders.converters.core.extensions import DEFAULT_EXTENSIONS, ExtensionRegistry, Extensions +from cdm_data_loaders.converters.core.extensions import ExtensionRegistry, Extensions, default_extension_registry from cdm_data_loaders.converters.core.ir import Field, NodeHints, NodeType, Provenance, SchemaDocument, TypedNode from cdm_data_loaders.converters.core.ir_values import Value, freeze_value @@ -77,11 +79,12 @@ def _metadata(value: Mapping[str, object]) -> Mapping[str, Value]: } -@dataclass(frozen=True, slots=True) -class IcebergReader: +class IcebergReader(BaseModel): """Read physical/logical types separately from JSON rendering conventions.""" - extension_registry: ExtensionRegistry = DEFAULT_EXTENSIONS + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) + + extension_registry: InstanceOf[ExtensionRegistry] = PydanticField(default_factory=default_extension_registry) mapping_target: str = "IR" def read(self, source: Schema) -> SchemaDocument: @@ -92,15 +95,17 @@ def read(self, source: Schema) -> SchemaDocument: return SchemaDocument( root=self._type(source.as_struct(), ()), extensions=Extensions( - { + payload={ "x-iceberg": { "schema_id": source.schema_id, "identifier_field_ids": sorted(source.identifier_field_ids), } }, - self.extension_registry, + registry=self.extension_registry, + ), + provenance=Provenance( + source="iceberg", metadata=_metadata(source.model_dump(mode="python", by_alias=True)) ), - provenance=Provenance("iceberg", metadata=_metadata(source.model_dump(mode="python", by_alias=True))), ) def read_table(self, table: Table, identifier: tuple[str, ...]) -> SchemaDocument: @@ -128,8 +133,10 @@ def read_table(self, table: Table, identifier: tuple[str, ...]) -> SchemaDocumen name=identifier[-1], identifier=f"urn:iceberg:{'.'.join(identifier)}", annotations={"description": table.properties["comment"]} if "comment" in table.properties else {}, - extensions=Extensions({"x-iceberg": iceberg}, self.extension_registry), - provenance=Provenance("iceberg", identifier, _metadata(metadata.model_dump(mode="python", by_alias=True))), + extensions=Extensions(payload={"x-iceberg": iceberg}, registry=self.extension_registry), + provenance=Provenance( + source="iceberg", path=identifier, metadata=_metadata(metadata.model_dump(mode="python", by_alias=True)) + ), ) def read_node(self, source: IcebergType) -> TypedNode: @@ -151,13 +158,15 @@ def _field(self, source: NestedField, path: tuple[str | int, ...]) -> Field: replace(self._type(source.field_type, path), nullable=not source.required), required=source.required, annotations={"description": source.doc} if "doc" in source.model_fields_set else {}, - extensions=Extensions({"x-iceberg": metadata}, self.extension_registry), - provenance=Provenance("iceberg", path, _metadata(source.model_dump(mode="python", by_alias=True))), + extensions=Extensions(payload={"x-iceberg": metadata}, registry=self.extension_registry), + provenance=Provenance( + source="iceberg", path=path, metadata=_metadata(source.model_dump(mode="python", by_alias=True)) + ), ) def _type(self, source: IcebergType, path: tuple[str | int, ...]) -> TypedNode: """Translate supported Iceberg types into genuine typed child trees.""" - provenance = Provenance("iceberg", path) + provenance = Provenance(source="iceberg", path=path) if type(source) in PRIMITIVES: kind, logical, width, timezone = PRIMITIVES[type(source)] return TypedNode( @@ -198,13 +207,13 @@ def _type(self, source: IcebergType, path: tuple[str | int, ...]) -> TypedNode: self._type(source.element_type, (*path, "element")), nullable=not source.element_required ), extensions=Extensions( - { + payload={ "x-iceberg": { "element_id": source.element_id, "element_required": source.element_required, } }, - self.extension_registry, + registry=self.extension_registry, ), ) if isinstance(source, MapType): @@ -215,14 +224,14 @@ def _type(self, source: IcebergType, path: tuple[str | int, ...]) -> TypedNode: key_type=self._type(source.key_type, (*path, "key")), value_type=replace(self._type(source.value_type, (*path, "value")), nullable=not source.value_required), extensions=Extensions( - { + payload={ "x-iceberg": { "key_id": source.key_id, "value_id": source.value_id, "value_required": source.value_required, } }, - self.extension_registry, + registry=self.extension_registry, ), ) msg = f"No {self.mapping_target} mapping for Iceberg type: {source}" diff --git a/src/cdm_data_loaders/converters/sources/json_schema/common.py b/src/cdm_data_loaders/converters/sources/json_schema/common.py index 21a83bf5..b6de3419 100644 --- a/src/cdm_data_loaders/converters/sources/json_schema/common.py +++ b/src/cdm_data_loaders/converters/sources/json_schema/common.py @@ -18,8 +18,8 @@ def extract_extensions(schema: Mapping[str, Any], registry: ExtensionRegistry) - :rtype: Extensions """ return Extensions( - {key: value for key, value in schema.items() if key.startswith("x-")}, - registry, + payload={key: value for key, value in schema.items() if key.startswith("x-")}, + registry=registry, ) diff --git a/src/cdm_data_loaders/converters/sources/json_schema/permissive.py b/src/cdm_data_loaders/converters/sources/json_schema/permissive.py index f9efd9c7..21762525 100644 --- a/src/cdm_data_loaders/converters/sources/json_schema/permissive.py +++ b/src/cdm_data_loaders/converters/sources/json_schema/permissive.py @@ -1,13 +1,14 @@ """Read dereferenced JSON Schema documents into the intermediate representation.""" from collections.abc import Mapping -from dataclasses import dataclass from typing import Any, cast from jsonschema.validators import validator_for +from pydantic import BaseModel, ConfigDict, InstanceOf +from pydantic import Field as PydanticField from cdm_data_loaders.converters.core.errors import ConversionError -from cdm_data_loaders.converters.core.extensions import DEFAULT_EXTENSIONS, ExtensionRegistry +from cdm_data_loaders.converters.core.extensions import ExtensionRegistry, default_extension_registry from cdm_data_loaders.converters.core.guards import ( reject_unresolved_references, require_schema_keyword, @@ -27,12 +28,13 @@ from cdm_data_loaders.validation.jsonschema.metaschema import ensure_custom_metaschema_registered -@dataclass(frozen=True, slots=True) -class JsonSchemaReader: +class JsonSchemaReader(BaseModel): """Read validated object-compatible JSON Schema documents using an extension registry.""" - extension_registry: ExtensionRegistry = DEFAULT_EXTENSIONS - error_type: type[ConversionError] = ConversionError + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) + + extension_registry: InstanceOf[ExtensionRegistry] = PydanticField(default_factory=default_extension_registry) + error_type: type[ConversionError] = PydanticField(default_factory=lambda: ConversionError) converter_name: str = "JsonSchemaReader" dereference_function: str = "dereference_schema" @@ -66,7 +68,7 @@ def read(self, source: Mapping[str, Any]) -> SchemaDocument: identifier=schema.get("$id"), annotations=node.annotations, extensions=node.extensions, - provenance=Provenance("json-schema"), + provenance=Provenance(source="json-schema"), jsonschema_validator_cls=validator_for(schema), ) @@ -90,7 +92,7 @@ def _node(self, schema: object, path: tuple[str | int, ...]) -> TypedNode: :return: Lowered ``TypedNode``. :rtype: TypedNode """ - provenance = Provenance("json-schema", path) + provenance = Provenance(source="json-schema", path=path) if isinstance(schema, bool): return TypedNode(type="any" if schema else "never", nullable=schema, provenance=provenance) if not isinstance(schema, Mapping): diff --git a/src/cdm_data_loaders/converters/sources/json_schema/strict_reader.py b/src/cdm_data_loaders/converters/sources/json_schema/strict_reader.py index a24b781f..d9650ee8 100644 --- a/src/cdm_data_loaders/converters/sources/json_schema/strict_reader.py +++ b/src/cdm_data_loaders/converters/sources/json_schema/strict_reader.py @@ -1,13 +1,15 @@ """Read strict Draft 2020-12 JSON Schema documents with ``$defs`` recursion into the IR.""" from collections.abc import Mapping -from dataclasses import dataclass, field, replace +from dataclasses import replace from typing import Any, NoReturn from frozendict import frozendict +from pydantic import BaseModel, ConfigDict, InstanceOf +from pydantic import Field as PydanticField -from cdm_data_loaders.converters.core.errors import ConversionError, raise_spec_error -from cdm_data_loaders.converters.core.extensions import DEFAULT_EXTENSIONS, ExtensionRegistry +from cdm_data_loaders.converters.core.errors import ConversionError, ErrorCodeMap, raise_spec_error +from cdm_data_loaders.converters.core.extensions import ExtensionRegistry, default_extension_registry from cdm_data_loaders.converters.core.guards import ( require_object_root, require_schema_keyword, @@ -40,7 +42,7 @@ validate_node, ) from cdm_data_loaders.converters.sources.json_schema.strict_scalars import resolve_scalar -from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH, validate_max_recursion_depth +from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH, StrictPositiveRecursionDepth from cdm_data_loaders.validation.jsonschema.keywords import strip_ref_target_identity from cdm_data_loaders.validation.jsonschema.metaschema import ( STRICT_SUPPORTED_SCHEMA_DIALECTS, @@ -48,20 +50,19 @@ ) -@dataclass(frozen=True, slots=True) -class StrictJsonSchemaReader: +class StrictJsonSchemaReader(BaseModel): """Read Draft 2020-12 JSON Schema documents under the strict tabular conversion specification.""" - extension_registry: ExtensionRegistry = DEFAULT_EXTENSIONS - error_type: type[ConversionError] = ConversionError - converter_name: str = "StrictJsonSchemaReader" - max_recursion_depth: int = DEFAULT_MAX_RECURSION_DEPTH - error_by_code: Mapping[str, type[ConversionError]] = field(default_factory=frozendict) - forbidden_recursive_formats: frozenset[str] = DEFAULT_FORBIDDEN_RECURSIVE_FORMATS + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) - def __post_init__(self) -> None: - """Validate reader recursion depth configuration.""" - validate_max_recursion_depth(self.max_recursion_depth) + extension_registry: InstanceOf[ExtensionRegistry] = PydanticField(default_factory=default_extension_registry) + error_type: type[ConversionError] = PydanticField(default_factory=lambda: ConversionError) + converter_name: str = "StrictJsonSchemaReader" + max_recursion_depth: StrictPositiveRecursionDepth = DEFAULT_MAX_RECURSION_DEPTH + error_by_code: ErrorCodeMap = PydanticField(default_factory=frozendict) + forbidden_recursive_formats: frozenset[str] = PydanticField( + default_factory=lambda: DEFAULT_FORBIDDEN_RECURSIVE_FORMATS + ) def _raise_err(self, code: str, msg: str) -> NoReturn: """Raise the configured exception subclass for ``code`` or fall back to ``error_type``. @@ -167,7 +168,7 @@ def read(self, source: Mapping[str, Any]) -> SchemaDocument: identifier=schema.get("$id"), annotations=node.annotations, extensions=node.extensions, - provenance=Provenance("json-schema"), + provenance=Provenance(source="json-schema"), jsonschema_validator_cls=validator_cls, ) @@ -295,15 +296,20 @@ def _build_merged_object( merged_prop, prop_ctx, (*meta.path, "properties", prop_name), required_in_parent=is_req ) fields.append( - Field(prop_name, prop_node, required=not bool(prop_node.nullable), provenance=prop_node.provenance) + Field( + name=prop_name, + node=prop_node, + required=not bool(prop_node.nullable), + provenance=prop_node.provenance, + ) ) if all_scalars: val_path = (*meta.path, "_value") val_type, val_hints, val_label = self._resolve_and_unify_scalar_branches(all_scalars, ctx, val_path) - val_prov = Provenance("json-schema", val_path, {"jsonschema_type": val_label}) + val_prov = Provenance(source="json-schema", path=val_path, metadata={"jsonschema_type": val_label}) val_node = TypedNode(type=val_type, nullable=True, hints=val_hints, provenance=val_prov) - fields.append(Field("_value", val_node, required=False, provenance=val_prov)) + fields.append(Field(name="_value", node=val_node, required=False, provenance=val_prov)) if not fields: msg = "Empty object schema with no properties cannot be converted to a PyArrow struct" @@ -318,7 +324,7 @@ def _build_merged_object( annotations=meta.annotations, extensions=meta.extensions, constraints=meta.constraints, - provenance=Provenance("json-schema", meta.path), + provenance=Provenance(source="json-schema", path=meta.path), ) def _build_map( @@ -350,7 +356,7 @@ def _build_map( type="string", nullable=False, hints=NodeHints(logical_type="string"), - provenance=Provenance("json-schema", meta.path, {"jsonschema_type": "string"}), + provenance=Provenance(source="json-schema", path=meta.path, metadata={"jsonschema_type": "string"}), ) return TypedNode( type="map", @@ -361,7 +367,7 @@ def _build_map( annotations=meta.annotations, extensions=meta.extensions, constraints=meta.constraints, - provenance=Provenance("json-schema", meta.path), + provenance=Provenance(source="json-schema", path=meta.path), ) def _build_array( @@ -393,8 +399,8 @@ def _build_array( ) fields.append( Field( - f"item_{idx}", - item_node, + name=f"item_{idx}", + node=item_node, required=not bool(item_node.nullable), provenance=item_node.provenance, ) @@ -412,7 +418,7 @@ def _build_array( annotations=meta.annotations, extensions=meta.extensions, constraints=meta.constraints, - provenance=Provenance("json-schema", meta.path), + provenance=Provenance(source="json-schema", path=meta.path), ) validated_items = self._validate(node.get("items")) @@ -425,7 +431,7 @@ def _build_array( annotations=meta.annotations, extensions=meta.extensions, constraints=meta.constraints, - provenance=Provenance("json-schema", meta.path), + provenance=Provenance(source="json-schema", path=meta.path), ) def _build_union_type( @@ -474,7 +480,7 @@ def _build_union_type( annotations=meta.annotations, extensions=meta.extensions, constraints=meta.constraints, - provenance=Provenance("json-schema", meta.path, {"jsonschema_type": u_label}), + provenance=Provenance(source="json-schema", path=meta.path, metadata={"jsonschema_type": u_label}), ) if obj_branches and not scalar_branches: @@ -520,7 +526,7 @@ def _build_ref_type( annotations=meta.annotations, extensions=meta.extensions, constraints=meta.constraints, - provenance=Provenance("json-schema", meta.path), + provenance=Provenance(source="json-schema", path=meta.path), ) sibling_keys = {k: v for k, v in validated.items() if k not in {"$ref", "description"}} if not sibling_keys: @@ -608,7 +614,7 @@ def _build_type( annotations=meta.annotations, extensions=meta.extensions, constraints=meta.constraints, - provenance=Provenance("json-schema", meta.path, {"jsonschema_type": s_label}), + provenance=Provenance(source="json-schema", path=meta.path, metadata={"jsonschema_type": s_label}), ) def _build_field_node( diff --git a/src/cdm_data_loaders/converters/sources/json_schema/strict_refs.py b/src/cdm_data_loaders/converters/sources/json_schema/strict_refs.py index 85c0fb75..0f4dbd7d 100644 --- a/src/cdm_data_loaders/converters/sources/json_schema/strict_refs.py +++ b/src/cdm_data_loaders/converters/sources/json_schema/strict_refs.py @@ -1,14 +1,18 @@ """Reference resolution, node validation, and cycle checks for strict JSON Schema.""" from collections.abc import Mapping -from dataclasses import dataclass, field, replace -from typing import Any, Final +from dataclasses import field, replace +from typing import Annotated, Any, Final from frozendict import frozendict +from pydantic import ConfigDict, GetPydanticSchema, InstanceOf, StrictBool, StrictInt, StrictStr +from pydantic.dataclasses import dataclass +from pydantic_core import core_schema -from cdm_data_loaders.converters.core.errors import ConversionError, raise_spec_error -from cdm_data_loaders.converters.core.extensions import DEFAULT_EXTENSIONS, ExtensionRegistry +from cdm_data_loaders.converters.core.errors import ConversionError, ErrorCodeMap, raise_spec_error +from cdm_data_loaders.converters.core.extensions import ExtensionRegistry, default_extension_registry from cdm_data_loaders.converters.sources.json_schema.common import extract_extensions +from cdm_data_loaders.utils.recursion import StrictPositiveRecursionDepth from cdm_data_loaders.validation.jsonschema.keywords import iter_child_subschemas CONSTRAINING_KEYWORDS: Final[frozenset[str]] = frozenset( @@ -33,17 +37,25 @@ DEFAULT_FORBIDDEN_RECURSIVE_FORMATS: Final[frozenset[str]] = frozenset({"date"}) -@dataclass(frozen=True, slots=True) +type _MappingInstance[K, V] = Annotated[ + Mapping[K, V], + GetPydanticSchema(lambda _tp, _handler: core_schema.is_instance_schema(Mapping)), +] + + +@dataclass(frozen=True, slots=True, config=ConfigDict(arbitrary_types_allowed=True, extra="forbid")) class StrictBuildContext: """Immutable traversal context for strict JSON Schema IR lowering.""" - defs: Mapping[str, Any] - ref_counts: Mapping[str, int] - max_recursion_depth: int - extension_registry: ExtensionRegistry = DEFAULT_EXTENSIONS - error_type: type[ConversionError] = ConversionError - error_by_code: Mapping[str, type[ConversionError]] = field(default_factory=frozendict) - forbidden_recursive_formats: frozenset[str] = DEFAULT_FORBIDDEN_RECURSIVE_FORMATS + defs: _MappingInstance[str, Any] + ref_counts: Mapping[StrictStr, StrictInt] + max_recursion_depth: StrictPositiveRecursionDepth + extension_registry: InstanceOf[ExtensionRegistry] = field(default_factory=default_extension_registry) + error_type: type[ConversionError] = field(default_factory=lambda: ConversionError) + error_by_code: ErrorCodeMap = field(default_factory=frozendict) + forbidden_recursive_formats: frozenset[StrictStr] = field( + default_factory=lambda: DEFAULT_FORBIDDEN_RECURSIVE_FORMATS + ) def with_ref(self, ref_name: str) -> "StrictBuildContext": """Return a new context with the recursion counter for ``ref_name`` incremented by 1. @@ -59,15 +71,15 @@ def with_ref(self, ref_name: str) -> "StrictBuildContext": ) -@dataclass(frozen=True, slots=True) +@dataclass(frozen=True, slots=True, config=ConfigDict(extra="forbid")) class StrictNodeMeta: """Node-level path, nullability, and facet payloads for strict IR lowering.""" - path: tuple[str | int, ...] = () - nullable: bool = False - annotations: Mapping[str, Any] = field(default_factory=frozendict) - extensions: Mapping[str, Any] = field(default_factory=frozendict) - constraints: Mapping[str, Any] = field(default_factory=frozendict) + path: tuple[StrictStr | StrictInt, ...] = () + nullable: StrictBool = False + annotations: _MappingInstance[str, Any] = field(default_factory=frozendict) + extensions: _MappingInstance[str, Any] = field(default_factory=frozendict) + constraints: _MappingInstance[str, Any] = field(default_factory=frozendict) def is_null_branch(node: Mapping[str, Any]) -> bool: diff --git a/src/cdm_data_loaders/converters/sources/pyarrow.py b/src/cdm_data_loaders/converters/sources/pyarrow.py index 99281bce..83ecf5c3 100644 --- a/src/cdm_data_loaders/converters/sources/pyarrow.py +++ b/src/cdm_data_loaders/converters/sources/pyarrow.py @@ -1,15 +1,16 @@ """Read PyArrow schemas, fields, and data types into the converter IR.""" from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field, replace +from dataclasses import replace from typing import Any, Final, Never import pyarrow as pa from frozendict import frozendict +from pydantic import BaseModel, ConfigDict, Field, InstanceOf, StrictBool from cdm_data_loaders.converters.core.constants import DECIMAL128_BITS, DECIMAL256_BITS -from cdm_data_loaders.converters.core.errors import ConversionError, raise_spec_error -from cdm_data_loaders.converters.core.extensions import DEFAULT_EXTENSIONS, ExtensionRegistry, Extensions +from cdm_data_loaders.converters.core.errors import ConversionError, ErrorCodeMap, raise_spec_error +from cdm_data_loaders.converters.core.extensions import ExtensionRegistry, Extensions, default_extension_registry from cdm_data_loaders.converters.core.ir import ( SPEC_BINARY, SPEC_BOOL, @@ -98,15 +99,16 @@ def _format_path(path: tuple[str | int, ...]) -> str: return ".".join(str(part) for part in path) if path else "" -@dataclass(frozen=True, slots=True) -class PyArrowReader: +class PyArrowReader(BaseModel): """Read PyArrow ``pa.Schema``, ``pa.Field``, and ``pa.DataType`` objects into the converter IR.""" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) + source_name: Source = "pyarrow" - extension_registry: ExtensionRegistry = DEFAULT_EXTENSIONS - include_pyarrow_metadata: bool = True - error_type: type[ConversionError] = PyArrowReaderError - error_by_code: Mapping[str, type[ConversionError]] = field(default_factory=frozendict) + extension_registry: InstanceOf[ExtensionRegistry] = Field(default_factory=default_extension_registry) + include_pyarrow_metadata: StrictBool = True + error_type: type[ConversionError] = Field(default_factory=lambda: PyArrowReaderError) + error_by_code: ErrorCodeMap = Field(default_factory=frozendict) def _raise_error(self, code: str, msg: str) -> Never: """Raise the configured exception class for ``code``. @@ -134,7 +136,7 @@ def _x_pyarrow(self, payload: Mapping[str, Any]) -> Mapping[str, Value]: """ if not self.include_pyarrow_metadata or not payload: return frozendict() - return Extensions({"x-pyarrow": dict(payload)}, self.extension_registry) + return Extensions(payload={"x-pyarrow": dict(payload)}, registry=self.extension_registry) def _validate_field_list( self, @@ -207,7 +209,7 @@ def read( required_names=required_names, additional_properties=TypedNode(type="never", nullable=False), annotations=annotations, - provenance=Provenance(self.source_name, ()), + provenance=Provenance(source=self.source_name), ) return SchemaDocument( root=root, @@ -215,7 +217,7 @@ def read( identifier=resolved_id, annotations=annotations, extensions=doc_extensions, - provenance=Provenance(self.source_name, ()), + provenance=Provenance(source=self.source_name), ) def read_field(self, source: pa.Field) -> SchemaField: @@ -284,7 +286,7 @@ def _field(self, pa_field: pa.Field, path: tuple[str | int, ...]) -> SchemaField required=not pa_field.nullable, annotations=annotations, extensions=field_extensions, - provenance=Provenance(self.source_name, path, prov_meta), + provenance=Provenance(source=self.source_name, path=path, metadata=prov_meta), ) def _item_node( @@ -397,7 +399,7 @@ def _type( """ meta = decode_field_metadata(metadata) prov_meta = {k: meta[k] for k in PROVENANCE_METADATA_KEYS if k in meta} - provenance = Provenance(self.source_name, path, prov_meta) + provenance = Provenance(source=self.source_name, path=path, metadata=prov_meta) loc = _format_path(path) if pa.types.is_null(dtype): diff --git a/src/cdm_data_loaders/converters/sources/pydantic/annotations.py b/src/cdm_data_loaders/converters/sources/pydantic/annotations.py index 8edcf217..e40dfc75 100644 --- a/src/cdm_data_loaders/converters/sources/pydantic/annotations.py +++ b/src/cdm_data_loaders/converters/sources/pydantic/annotations.py @@ -15,7 +15,6 @@ Sequence, ) from collections.abc import Set as AbstractSet -from dataclasses import dataclass from enum import StrEnum from typing import ( Annotated, @@ -31,7 +30,8 @@ get_origin, ) -from pydantic import Base64Bytes, BaseModel, NaiveDatetime, RootModel +from pydantic import Base64Bytes, BaseModel, ConfigDict, NaiveDatetime, RootModel, StrictBool, StrictInt, StrictStr +from pydantic.dataclasses import dataclass as pydantic_dataclass from pydantic.fields import FieldInfo from pydantic_core import PydanticUndefined @@ -88,16 +88,16 @@ ) -@dataclass(frozen=True, slots=True) +@pydantic_dataclass(frozen=True, slots=True, config=ConfigDict(extra="forbid")) class ParsedAnnotation: """Unwrapped annotation paired with extracted Pydantic metadata.""" annotation: Any - description: str | None = None - max_digits: int | None = None - decimal_places: int | None = None - is_naive_datetime: bool = False - is_base64_bytes: bool = False + description: StrictStr | None = None + max_digits: StrictInt | None = None + decimal_places: StrictInt | None = None + is_naive_datetime: StrictBool = False + is_base64_bytes: StrictBool = False def is_union_origin(origin: object) -> bool: diff --git a/src/cdm_data_loaders/converters/sources/pydantic/model_synthesis.py b/src/cdm_data_loaders/converters/sources/pydantic/model_synthesis.py index 5e29bce9..6159a976 100644 --- a/src/cdm_data_loaders/converters/sources/pydantic/model_synthesis.py +++ b/src/cdm_data_loaders/converters/sources/pydantic/model_synthesis.py @@ -3,37 +3,61 @@ import inspect import sys from collections.abc import Mapping -from dataclasses import MISSING, dataclass, field, is_dataclass +from dataclasses import MISSING, field, is_dataclass, replace from dataclasses import fields as dc_fields from enum import Enum, IntEnum, StrEnum -from typing import Any, NoReturn, NotRequired, Required, Self, get_args, get_origin, get_type_hints, is_typeddict +from typing import ( + Annotated, + Any, + NoReturn, + NotRequired, + Required, + Self, + get_args, + get_origin, + get_type_hints, + is_typeddict, +) from frozendict import frozendict from pydantic import ( BaseModel, + ConfigDict, Field, + GetPydanticSchema, + InstanceOf, PydanticSchemaGenerationError, PydanticUndefinedAnnotation, RootModel, + StrictBool, + StrictInt, + StrictStr, create_model, ) +from pydantic.dataclasses import dataclass as pydantic_dataclass from pydantic.fields import FieldInfo +from pydantic_core import core_schema + +from cdm_data_loaders.converters.core.errors import ConversionError, ErrorCodeMap, raise_spec_error +from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH, StrictPositiveRecursionDepth -from cdm_data_loaders.converters.core.errors import ConversionError, raise_spec_error -from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH +type _ModuleNamespace = Annotated[ + Mapping[str, Any], + GetPydanticSchema(lambda _tp, _handler: core_schema.is_instance_schema(Mapping)), +] -@dataclass(frozen=True, slots=True) +@pydantic_dataclass(frozen=True, slots=True, config=ConfigDict(arbitrary_types_allowed=True, extra="forbid")) class SchemaBuildContext: """Immutable traversal context for Pydantic to IR schema generation.""" - model_counts: Mapping[type[BaseModel], int] = field(default_factory=frozendict) - active_aliases: frozenset[object] = field(default_factory=frozenset) - max_recursion_depth: int = DEFAULT_MAX_RECURSION_DEPTH - by_alias: bool = False - module_ns: Mapping[str, Any] = field(default_factory=frozendict) - error_type: type[ConversionError] = ConversionError - error_by_code: Mapping[str, type[ConversionError]] = field(default_factory=frozendict) + model_counts: Mapping[type[BaseModel], StrictInt] = field(default_factory=frozendict) + active_aliases: frozenset[Any] = field(default_factory=frozenset) + max_recursion_depth: StrictPositiveRecursionDepth = DEFAULT_MAX_RECURSION_DEPTH + by_alias: StrictBool = False + module_ns: _ModuleNamespace = field(default_factory=frozendict) + error_type: type[ConversionError] = field(default_factory=lambda: ConversionError) + error_by_code: ErrorCodeMap = field(default_factory=frozendict) def raise_err(self, code: str, msg: str, *, cause: BaseException | None = None) -> NoReturn: """Raise the configured exception subclass for ``code`` or fall back to ``error_type``. @@ -64,14 +88,11 @@ def with_model(self, model: type[BaseModel]) -> Self: """ mod = sys.modules.get(model.__module__) ns = frozendict(vars(mod)) if mod is not None else self.module_ns - return type(self)( + return replace( + self, model_counts=frozendict({**self.model_counts, model: self.model_counts.get(model, 0) + 1}), active_aliases=frozenset(), - max_recursion_depth=self.max_recursion_depth, - by_alias=self.by_alias, module_ns=ns, - error_type=self.error_type, - error_by_code=self.error_by_code, ) def with_alias(self, alias_obj: object) -> Self: @@ -82,27 +103,19 @@ def with_alias(self, alias_obj: object) -> Self: :return: Updated traversal context. :rtype: Self """ - return type(self)( - model_counts=self.model_counts, - active_aliases=self.active_aliases | {alias_obj}, - max_recursion_depth=self.max_recursion_depth, - by_alias=self.by_alias, - module_ns=self.module_ns, - error_type=self.error_type, - error_by_code=self.error_by_code, - ) + return replace(self, active_aliases=self.active_aliases | {alias_obj}) -@dataclass(frozen=True, slots=True) +@pydantic_dataclass(frozen=True, slots=True, config=ConfigDict(extra="forbid")) class ModelFieldSpec: """Internal specification of an active field extracted from a ``BaseModel``.""" - output_name: str - annotation: object - field_info: FieldInfo | None - description: str | None - required_non_null: bool - context: SchemaBuildContext + output_name: StrictStr + annotation: Any + field_info: InstanceOf[FieldInfo] | None + description: StrictStr | None + required_non_null: StrictBool + context: InstanceOf[SchemaBuildContext] def ensure_model_complete(model: type[BaseModel], ctx: SchemaBuildContext | None = None) -> None: diff --git a/src/cdm_data_loaders/converters/sources/pydantic/reader.py b/src/cdm_data_loaders/converters/sources/pydantic/reader.py index 7195b761..7613c567 100644 --- a/src/cdm_data_loaders/converters/sources/pydantic/reader.py +++ b/src/cdm_data_loaders/converters/sources/pydantic/reader.py @@ -1,14 +1,13 @@ """Public ``PydanticReader`` lowering Pydantic v2 ``BaseModel`` classes into the converter IR.""" import sys -from collections.abc import Mapping -from dataclasses import dataclass, field, replace +from dataclasses import replace from typing import NoReturn, get_origin from frozendict import frozendict -from pydantic import BaseModel, RootModel +from pydantic import BaseModel, ConfigDict, Field, RootModel, StrictBool -from cdm_data_loaders.converters.core.errors import ConversionError, raise_spec_error +from cdm_data_loaders.converters.core.errors import ConversionError, ErrorCodeMap, raise_spec_error from cdm_data_loaders.converters.core.inference import unify_scalar_specs from cdm_data_loaders.converters.core.ir import Provenance, SchemaDocument, TypedNode from cdm_data_loaders.converters.sources.pydantic.annotations import ( @@ -26,11 +25,10 @@ ) from cdm_data_loaders.converters.sources.pydantic.scalars import resolve_parsed_scalar from cdm_data_loaders.converters.sources.pydantic.unions import build_merged_models_struct -from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH, validate_max_recursion_depth +from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH, StrictPositiveRecursionDepth -@dataclass(frozen=True, slots=True) -class PydanticReader: +class PydanticReader(BaseModel): """Build typed converter IR documents from Pydantic v2 ``BaseModel`` classes. :param max_recursion_depth: Maximum times a recursive ``BaseModel`` may be expanded along an ancestor path. @@ -40,20 +38,15 @@ class PydanticReader: :param error_type: Base exception class raised when a construct cannot be converted. :type error_type: type[ConversionError] :param error_by_code: Mapping from specification error code (``E1``-``E12``) to exception class. - :type error_by_code: Mapping[str, type[ConversionError]] + :type error_by_code: ErrorCodeMap """ - max_recursion_depth: int = DEFAULT_MAX_RECURSION_DEPTH - by_alias: bool = False - error_type: type[ConversionError] = ConversionError - error_by_code: Mapping[str, type[ConversionError]] = field(default_factory=frozendict) + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) - def __post_init__(self) -> None: - """Validate reader configuration.""" - validate_max_recursion_depth(self.max_recursion_depth) - if type(self.by_alias) is not bool: - msg = "by_alias must be a boolean" - raise TypeError(msg) + max_recursion_depth: StrictPositiveRecursionDepth = DEFAULT_MAX_RECURSION_DEPTH + by_alias: StrictBool = False + error_type: type[ConversionError] = Field(default_factory=lambda: ConversionError) + error_by_code: ErrorCodeMap = Field(default_factory=frozendict) def _raise_err(self, code: str, msg: str) -> NoReturn: """Raise the configured exception subclass for ``code`` or fall back to ``error_type``. @@ -130,7 +123,7 @@ def read(self, source: type[BaseModel]) -> SchemaDocument: name=source.__name__, identifier=f"urn:pydantic:{source.__module__}.{qualname}", annotations=annotations, - provenance=Provenance("pydantic", (source.__name__,)), + provenance=Provenance(source="pydantic", path=(source.__name__,)), ) def read_scalar_annotation(self, annotation: object) -> TypedNode: @@ -163,5 +156,5 @@ def read_scalar_annotation(self, annotation: object) -> TypedNode: type=node_type, nullable=allows_null(annotation, ctx), hints=hints, - provenance=Provenance("pydantic", (), {"pydantic_type": pyd_type}), + provenance=Provenance(source="pydantic", path=(), metadata={"pydantic_type": pyd_type}), ) diff --git a/src/cdm_data_loaders/converters/sources/pydantic/unions.py b/src/cdm_data_loaders/converters/sources/pydantic/unions.py index fae0ca5c..54712a89 100644 --- a/src/cdm_data_loaders/converters/sources/pydantic/unions.py +++ b/src/cdm_data_loaders/converters/sources/pydantic/unions.py @@ -63,7 +63,7 @@ def build_sequence_node( type="array", nullable=nullable, items=item_node, - provenance=Provenance("pydantic", path), + provenance=Provenance(source="pydantic", path=path), ) @@ -106,7 +106,7 @@ def build_tuple_node( properties=fields, required_names=required_names, additional_properties=TypedNode(type="never", nullable=False), - provenance=Provenance("pydantic", path), + provenance=Provenance(source="pydantic", path=path), ) @@ -142,7 +142,7 @@ def build_map_node( type="string", nullable=False, hints=NodeHints(logical_type="string"), - provenance=Provenance("pydantic", (*path, "key")), + provenance=Provenance(source="pydantic", path=(*path, "key")), ) val_field = build_field_ir("value", val_ann, ctx, (*path, "value"), required_in_parent=True) val_node = replace(val_field.node, annotations=val_field.annotations) @@ -151,7 +151,7 @@ def build_map_node( nullable=nullable, key_type=key_node, value_type=val_node, - provenance=Provenance("pydantic", path), + provenance=Provenance(source="pydantic", path=path), ) @@ -250,7 +250,7 @@ def build_merged_models_struct( node=node, required=not field_nullable, annotations=annotations, - provenance=Provenance("pydantic", field_path, prov_meta), + provenance=Provenance(source="pydantic", path=field_path, metadata=prov_meta), ) ) @@ -258,7 +258,7 @@ def build_merged_models_struct( resolved_scalars = [resolve_parsed_scalar(sb, ctx) for sb in scalar_branches] val_node_type, val_hints, val_label = unify_scalar_specs(resolved_scalars) val_path = (*path, "_value") - val_prov = Provenance("pydantic", val_path, {"pydantic_type": val_label}) + val_prov = Provenance(source="pydantic", path=val_path, metadata={"pydantic_type": val_label}) val_node = TypedNode(type=val_node_type, nullable=True, hints=val_hints, provenance=val_prov) ir_fields.append(SchemaField(name="_value", node=val_node, required=False, provenance=val_prov)) @@ -281,7 +281,7 @@ def build_merged_models_struct( properties=fields_tuple, required_names=required_names, additional_properties=TypedNode(type="never", nullable=False), - provenance=Provenance("pydantic", path, prov_meta), + provenance=Provenance(source="pydantic", path=path, metadata=prov_meta), ) @@ -391,7 +391,7 @@ def build_multi_branch_node( type=node_type, nullable=nullable, hints=hints, - provenance=Provenance("pydantic", path, {"pydantic_type": pyd_type}), + provenance=Provenance(source="pydantic", path=path, metadata={"pydantic_type": pyd_type}), ) for m in model_branches: @@ -404,7 +404,7 @@ def build_multi_branch_node( type="string", nullable=nullable, hints=NodeHints(logical_type="string"), - provenance=Provenance("pydantic", path), + provenance=Provenance(source="pydantic", path=path), ) return build_merged_models_struct( @@ -454,7 +454,7 @@ def build_parsed_node( type="string", nullable=nullable, hints=NodeHints(logical_type="string"), - provenance=Provenance("pydantic", path), + provenance=Provenance(source="pydantic", path=path), ) return build_merged_models_struct([model_cls], ctx, path, nullable=nullable) builder = CONTAINER_NODE_BUILDERS.get(kind) @@ -466,7 +466,7 @@ def build_parsed_node( type=node_type, nullable=nullable, hints=hints, - provenance=Provenance("pydantic", path, {"pydantic_type": pyd_type}), + provenance=Provenance(source="pydantic", path=path, metadata={"pydantic_type": pyd_type}), ) @@ -502,7 +502,7 @@ def build_parsed_field_ir( node=node, required=not nullable, annotations=annotations, - provenance=Provenance("pydantic", path, prov_meta), + provenance=Provenance(source="pydantic", path=path, metadata=prov_meta), ) diff --git a/src/cdm_data_loaders/converters/sources/xsd.py b/src/cdm_data_loaders/converters/sources/xsd.py index 8c107a1e..6609304b 100644 --- a/src/cdm_data_loaders/converters/sources/xsd.py +++ b/src/cdm_data_loaders/converters/sources/xsd.py @@ -1,19 +1,20 @@ """Read XML Schema Definition (XSD) documents into the converter IR.""" from collections.abc import Mapping -from dataclasses import dataclass, field, replace +from dataclasses import replace from pathlib import Path from typing import Any, Final, NoReturn import xmlschema from frozendict import frozendict +from pydantic import BaseModel, ConfigDict, Field from xmlschema.validators import ( XsdComplexType, XsdElement, XsdType, ) -from cdm_data_loaders.converters.core.errors import ConversionError, raise_spec_error +from cdm_data_loaders.converters.core.errors import ConversionError, ErrorCodeMap, raise_spec_error from cdm_data_loaders.converters.core.ir import ( SPEC_BINARY, SPEC_BOOL, @@ -35,7 +36,7 @@ TypedNode, ) from cdm_data_loaders.converters.core.ir import Field as SchemaField -from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH, validate_max_recursion_depth +from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH, StrictPositiveRecursionDepth from cdm_data_loaders.utils.xml_utils import local_name as _local_name from cdm_data_loaders.validation.xsd.particles import resolve_child_cardinality from cdm_data_loaders.validation.xsd.scalars import ( @@ -106,7 +107,7 @@ def _string_ir_node( type="string", nullable=nullable, hints=NodeHints(logical_type="string"), - provenance=Provenance("xsd", path, metadata or {}), + provenance=Provenance(source="xsd", path=path, metadata=metadata or {}), ) @@ -153,13 +154,13 @@ def simple_type_to_ir_node( type=item_nt, nullable=True, hints=item_hints, - provenance=Provenance("xsd", (*path, "item"), {"xsd_type": item_xsd}), + provenance=Provenance(source="xsd", path=(*path, "item"), metadata={"xsd_type": item_xsd}), ) return TypedNode( type="array", nullable=nullable, items=item_node, - provenance=Provenance("xsd", path), + provenance=Provenance(source="xsd", path=path), ) node_type, hints, xsd_name = _resolve_scalar_ir_spec(simple_type) @@ -167,7 +168,7 @@ def simple_type_to_ir_node( type=node_type, nullable=nullable, hints=hints, - provenance=Provenance("xsd", path, {"xsd_type": xsd_name}), + provenance=Provenance(source="xsd", path=path, metadata={"xsd_type": xsd_name}), ) @@ -242,7 +243,7 @@ def _wrap_leaf_ir_field( type="array", nullable=nullable, items=leaf_node, - provenance=Provenance("xsd", path), + provenance=Provenance(source="xsd", path=path), ) prov_meta: dict[str, Any] = {} else: @@ -254,7 +255,7 @@ def _wrap_leaf_ir_field( node=field_node, required=not nullable, annotations=annotations, - provenance=Provenance("xsd", path, prov_meta), + provenance=Provenance(source="xsd", path=path, metadata=prov_meta), ) @@ -292,7 +293,7 @@ def _wrap_truncated_leaf_ir_field( type="array", nullable=nullable, items=leaf_node, - provenance=Provenance("xsd", path, trunc_meta), + provenance=Provenance(source="xsd", path=path, metadata=trunc_meta), ) if is_list else leaf_node @@ -302,7 +303,7 @@ def _wrap_truncated_leaf_ir_field( node=field_node, required=not nullable, annotations=annotations, - provenance=Provenance("xsd", path, trunc_meta), + provenance=Provenance(source="xsd", path=path, metadata=trunc_meta), ) @@ -349,7 +350,7 @@ def _build_complex_struct_fields( node=node, required=not nullable, annotations=annotations, - provenance=Provenance("xsd", attr_path, prov_meta), + provenance=Provenance(source="xsd", path=attr_path, metadata=prov_meta), ) ) @@ -367,7 +368,7 @@ def _build_complex_struct_fields( name="#text", node=node, required=False, - provenance=Provenance("xsd", text_path, prov_meta), + provenance=Provenance(source="xsd", path=text_path, metadata=prov_meta), ) ) return tuple(fields) @@ -460,7 +461,7 @@ def _build_element_ir_field( properties=struct_fields, required_names=required_names, additional_properties=TypedNode(type="never", nullable=False), - provenance=Provenance("xsd", path, struct_meta), + provenance=Provenance(source="xsd", path=path, metadata=struct_meta), ) if is_cacheable: ctx.struct_cache[cache_key] = struct_node @@ -470,7 +471,7 @@ def _build_element_ir_field( type="array", nullable=nullable, items=replace(struct_node, nullable=True), - provenance=Provenance("xsd", path), + provenance=Provenance(source="xsd", path=path), ) if is_list else replace(struct_node, nullable=nullable) @@ -481,7 +482,7 @@ def _build_element_ir_field( node=field_node, required=not nullable, annotations=annotations, - provenance=Provenance("xsd", path), + provenance=Provenance(source="xsd", path=path), ) @@ -531,7 +532,7 @@ def _build_element_struct_node( properties=fields, required_names=required_names, additional_properties=TypedNode(type="never", nullable=False), - provenance=Provenance("xsd", (), struct_meta), + provenance=Provenance(source="xsd", path=(), metadata=struct_meta), ) @@ -561,7 +562,7 @@ def _finish_element_doc( name=_local_name(tag_name), identifier=schema.target_namespace or None, annotations=annotations, - provenance=Provenance("xsd", (tag_name,)), + provenance=Provenance(source="xsd", path=(tag_name,)), ) @@ -600,7 +601,7 @@ def _build_element_table_doc( properties=(text_field,), required_names=(), additional_properties=TypedNode(type="never", nullable=False), - provenance=Provenance("xsd", (), {"scalar_leaf_table": True}), + provenance=Provenance(source="xsd", path=(), metadata={"scalar_leaf_table": True}), ) else: elem_type = require_complex_type( @@ -615,8 +616,7 @@ def _build_element_table_doc( return _finish_element_doc(base_ctx.schema, target_elem, tag_name, root_node) -@dataclass(frozen=True, slots=True) -class XsdReader: +class XsdReader(BaseModel): """Build typed converter IR documents from XML Schema Definition (XSD) files. :param xml_tag: Default target XML element tag name when not supplied to :meth:`read`. @@ -626,17 +626,15 @@ class XsdReader: :param error_type: Base exception class raised when a construct cannot be converted. :type error_type: type[ConversionError] :param error_by_code: Mapping from specification error code (``E1``-``E9``) to exception class. - :type error_by_code: Mapping[str, type[ConversionError]] + :type error_by_code: ErrorCodeMap """ - xml_tag: str | None = None - max_recursion_depth: int = DEFAULT_MAX_RECURSION_DEPTH - error_type: type[ConversionError] = ConversionError - error_by_code: Mapping[str, type[ConversionError]] = field(default_factory=frozendict) + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid", frozen=True) - def __post_init__(self) -> None: - """Validate reader configuration.""" - validate_max_recursion_depth(self.max_recursion_depth) + xml_tag: str | None = None + max_recursion_depth: StrictPositiveRecursionDepth = DEFAULT_MAX_RECURSION_DEPTH + error_type: type[ConversionError] = Field(default_factory=lambda: ConversionError) + error_by_code: ErrorCodeMap = Field(default_factory=frozendict) def _raise_err(self, code: str, msg: str) -> NoReturn: """Raise the configured exception subclass for ``code`` or fall back to ``error_type``. diff --git a/src/cdm_data_loaders/core/reporting.py b/src/cdm_data_loaders/core/reporting.py index 6d1429fc..0bdcb977 100644 --- a/src/cdm_data_loaders/core/reporting.py +++ b/src/cdm_data_loaders/core/reporting.py @@ -1,15 +1,16 @@ """Reporting and output for pipelines.""" -from dataclasses import dataclass from datetime import datetime from logging import Logger, getLogger from pathlib import Path -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Annotated, Final import dlt import yaml from dlt.common.pipeline import LoadInfo from dlt.common.runtime.slack import send_slack_message +from pydantic import BaseModel, ConfigDict, GetPydanticSchema +from pydantic_core import core_schema from cdm_data_loaders.core.fields import JSONL from cdm_data_loaders.core.settings import CtsSettings @@ -23,16 +24,22 @@ logger: Logger = getLogger(__name__) +type _LoadInfoField = Annotated[ + LoadInfo, + GetPydanticSchema(lambda _tp, _handler: core_schema.is_instance_schema(LoadInfo)), +] -@dataclass(frozen=True, slots=True) -class PipelineReport: + +class PipelineReport(BaseModel): """Summary information for a pipeline run report.""" + model_config = ConfigDict(extra="forbid", frozen=True) + pipeline_name: str - dataset_name: str + dataset_name: str | None started_at: datetime - finished_at: datetime - pipeline_load_info: LoadInfo | None = None + finished_at: datetime | None = None + pipeline_load_info: _LoadInfoField | None = None pipeline_err_msg: str | None = None info_load_err_msg: str | None = None ingested_files: list[str] | None = None diff --git a/src/cdm_data_loaders/datasets/ncbi_ftp/promote.py b/src/cdm_data_loaders/datasets/ncbi_ftp/promote.py index 4fc52ca2..51728c58 100644 --- a/src/cdm_data_loaders/datasets/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/datasets/ncbi_ftp/promote.py @@ -4,13 +4,13 @@ import tempfile from collections import defaultdict from concurrent.futures import ThreadPoolExecutor, as_completed -from dataclasses import dataclass from datetime import UTC, datetime from logging import Logger, getLogger from pathlib import Path, PurePosixPath from typing import Any, Final, TypedDict, Unpack import tqdm +from pydantic import BaseModel, ConfigDict from cdm_data_loaders.datasets.ncbi_ftp.assembly import ( build_accession_path, @@ -47,10 +47,11 @@ MAX_TRANSFER_WORKERS: Final[int] = 32 -@dataclass(frozen=True, slots=True) -class _PromoteTargets: +class _PromoteTargets(BaseModel): """Source staging and destination Lakehouse S3 bucket/prefix configuration.""" + model_config = ConfigDict(extra="forbid", frozen=True) + staging_bucket: PurePosixPath staging_prefix: PurePosixPath lakehouse_bucket: PurePosixPath diff --git a/src/cdm_data_loaders/datasets/ncbi_ftp/summary.py b/src/cdm_data_loaders/datasets/ncbi_ftp/summary.py index a253957d..fa2fe94f 100644 --- a/src/cdm_data_loaders/datasets/ncbi_ftp/summary.py +++ b/src/cdm_data_loaders/datasets/ncbi_ftp/summary.py @@ -2,13 +2,15 @@ import csv from collections.abc import Iterable -from dataclasses import dataclass, field +from dataclasses import dataclass from logging import Logger, getLogger from pathlib import Path, PurePosixPath from types import MappingProxyType from typing import Final from urllib.parse import urlsplit +from pydantic import BaseModel, ConfigDict, Field + from cdm_data_loaders.datasets.ncbi_ftp.constants import ACCESSION_PARTS_REGEX, FTP_HOST, Source from cdm_data_loaders.io.ftp_client import ftp_retrieve_text, ftp_session @@ -28,7 +30,7 @@ MIN_COL: Final[int] = 20 -@dataclass +@dataclass(slots=True) class AssemblyRecord: """Parsed row from an NCBI assembly summary file.""" @@ -39,14 +41,15 @@ class AssemblyRecord: assembly_dir: PurePosixPath -@dataclass -class DiffResult: +class DiffResult(BaseModel): """Result of comparing two NCBI assembly summary states.""" - new: list[str] = field(default_factory=list) - updated: list[str] = field(default_factory=list) - replaced: list[str] = field(default_factory=list) - suppressed: list[str] = field(default_factory=list) + model_config = ConfigDict(extra="forbid", frozen=True) + + new: list[str] = Field(default_factory=list) + updated: list[str] = Field(default_factory=list) + replaced: list[str] = Field(default_factory=list) + suppressed: list[str] = Field(default_factory=list) def ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> PurePosixPath: diff --git a/src/cdm_data_loaders/datasets/uniprot/kb.py b/src/cdm_data_loaders/datasets/uniprot/uniprot_kb.py similarity index 100% rename from src/cdm_data_loaders/datasets/uniprot/kb.py rename to src/cdm_data_loaders/datasets/uniprot/uniprot_kb.py diff --git a/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py b/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py index 1364641b..8915a423 100644 --- a/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py +++ b/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py @@ -8,7 +8,10 @@ from cdm_data_loaders.core.cli import CliRunner from cdm_data_loaders.core.fields import LogInterval from cdm_data_loaders.core.transforms.settings_mixin import LockedTransformHooksMixin -from cdm_data_loaders.datasets.uniprot.kb import UNIPROT_CDM_SCHEMA_MAP, make_uniprot_cdm_transform +from cdm_data_loaders.datasets.uniprot.uniprot_kb import ( + UNIPROT_CDM_SCHEMA_MAP, + make_uniprot_cdm_transform, +) from cdm_data_loaders.datasets.uniprot.uniref import UNIREF_CDM_SCHEMA_MAP, make_uniref_cdm_transform from cdm_data_loaders.pipelines.presets.uniprot_xml import ( UniProtXmlToPyArrowSettings, diff --git a/src/cdm_data_loaders/pipelines/runner.py b/src/cdm_data_loaders/pipelines/runner.py index d1091116..cc42ea61 100644 --- a/src/cdm_data_loaders/pipelines/runner.py +++ b/src/cdm_data_loaders/pipelines/runner.py @@ -99,8 +99,8 @@ def run_pipeline( err_msg = f"Pipeline failed: {e!s}" logger.exception(err_msg) - has_errors = bool(load_info and load_info.has_failed_jobs) or bool(collector.preprocess_errors) - if load_info: + has_errors = bool(load_info is not None and load_info.has_failed_jobs) or bool(collector.preprocess_errors) + if load_info is not None: logger.info(load_info) if has_errors: logger.warning("Pipeline contains failed jobs") diff --git a/src/cdm_data_loaders/pipelines/xsv/pipeline.py b/src/cdm_data_loaders/pipelines/xsv/pipeline.py index 44d9770d..f43c8a9b 100644 --- a/src/cdm_data_loaders/pipelines/xsv/pipeline.py +++ b/src/cdm_data_loaders/pipelines/xsv/pipeline.py @@ -10,7 +10,6 @@ import gzip import shutil from collections.abc import Generator -from dataclasses import dataclass from pathlib import Path from tempfile import TemporaryDirectory, mkdtemp from typing import Any @@ -21,6 +20,7 @@ from dlt.common.storages.fsspec_filesystem import FileItemDict from dlt.common.typing import TDataItems from dlt.sources.filesystem import filesystem +from pydantic import BaseModel, ConfigDict from cdm_data_loaders.core.buffer import DictBuffer from cdm_data_loaders.core.cli import CliRunner @@ -41,8 +41,7 @@ from cdm_data_loaders.validation.jsonschema.models import generate_header -@dataclass -class XsvWorkPaths: +class XsvWorkPaths(BaseModel): """Scratch resources shared across all files processed in one pipeline run. :param qsv_cmd: path to the qsv binary @@ -59,6 +58,8 @@ class XsvWorkPaths: :type download_dir: Path """ + model_config = ConfigDict(extra="forbid", frozen=True) + qsv_cmd: str header_file_path: Path tmp_dir: Path diff --git a/src/cdm_data_loaders/readers/xsv/models.py b/src/cdm_data_loaders/readers/xsv/models.py index 28c9925f..35966ad1 100644 --- a/src/cdm_data_loaders/readers/xsv/models.py +++ b/src/cdm_data_loaders/readers/xsv/models.py @@ -3,7 +3,6 @@ import os import subprocess from collections.abc import Callable -from dataclasses import dataclass from functools import cached_property from pathlib import Path from shutil import copy, move @@ -11,6 +10,7 @@ from pydantic import ( BaseModel, + ConfigDict, DirectoryPath, Field, FilePath, @@ -298,15 +298,25 @@ def prepend_header(header_file_path: Path, xsv_file_path: Path, dest: Path) -> N out.write(xsv_file_path.read_bytes()) -@dataclass(slots=True) -class FileNames: +class FileNames(BaseModel): """Derived file paths produced during a ``qsv validate`` pass.""" + model_config = ConfigDict(extra="forbid", frozen=True, validate_by_name=True, validate_by_alias=True) + valid_output: Path errors: Path valid_lines: Path invalid_lines: Path - schema: Path + schema_path: Path = Field(alias="schema") + + @property + def schema(self) -> Path: # type: ignore[override] + """Return the derived JSON Schema file path for this validation pass. + + :return: Derived JSON Schema path. + :rtype: Path + """ + return self.schema_path def generate_qsv_validate_file_names( @@ -323,7 +333,7 @@ def generate_qsv_validate_file_names( :type input_file_name: str :param first_pass: Whether this is the first validation pass (defaults to ``True``). :type first_pass: bool - :return: Dataclass containing derived output, error, valid/invalid split, and schema paths. + :return: Model containing derived output, error, valid/invalid split, and schema paths. :rtype: FileNames """ output_file_stem = ( diff --git a/src/cdm_data_loaders/utils/checksums.py b/src/cdm_data_loaders/utils/checksums.py index 60f2f0de..a095c5b4 100644 --- a/src/cdm_data_loaders/utils/checksums.py +++ b/src/cdm_data_loaders/utils/checksums.py @@ -3,12 +3,12 @@ import base64 import hashlib import re -from dataclasses import dataclass from logging import Logger, getLogger from pathlib import Path, PurePosixPath from typing import IO, Any, Final, Protocol, runtime_checkable from awscrt.checksums import crc64nvme as _crc64nvme +from pydantic import BaseModel, ConfigDict HASH_CHUNK_BYTES: Final[int] = 1 << 20 @@ -21,8 +21,7 @@ logger: Logger = getLogger(__name__) -@dataclass(frozen=True, slots=True) -class ChecksumEntry: +class ChecksumEntry(BaseModel): """A single expected checksum, tied to a specific algorithm. :param algorithm: hashlib algorithm name the checksum was generated with @@ -31,6 +30,8 @@ class ChecksumEntry: :type value: str """ + model_config = ConfigDict(extra="forbid", frozen=True) + algorithm: str value: str diff --git a/src/cdm_data_loaders/utils/recursion.py b/src/cdm_data_loaders/utils/recursion.py index 0cc8ba53..61829715 100644 --- a/src/cdm_data_loaders/utils/recursion.py +++ b/src/cdm_data_loaders/utils/recursion.py @@ -2,7 +2,7 @@ from typing import Annotated, Final -from pydantic import BeforeValidator +from pydantic import Field, PositiveInt DEFAULT_MAX_RECURSION_DEPTH: Final[int] = 2 @@ -41,19 +41,4 @@ def validate_max_recursion_depth(depth: int) -> int: return validate_positive_int_param(depth, "max_recursion_depth") -def _pydantic_positive_recursion_depth(value: object) -> int: - """Validate ``value`` for Pydantic fields, converting ``TypeError`` into ``ValueError``. - - :param value: Candidate recursion depth value supplied to a Pydantic model. - :type value: object - :return: Validated integer recursion depth ``>= 1``. - :rtype: int - :raises ValueError: If ``value`` is a ``bool``, non-integer, or less than ``1``. - """ - try: - return validate_max_recursion_depth(value) # type: ignore[arg-type] - except TypeError as exc: - raise ValueError(str(exc)) from exc - - -StrictPositiveRecursionDepth = Annotated[int, BeforeValidator(_pydantic_positive_recursion_depth)] +StrictPositiveRecursionDepth = Annotated[PositiveInt, Field(strict=True)] diff --git a/src/cdm_data_loaders/validation/xsd/traversal.py b/src/cdm_data_loaders/validation/xsd/traversal.py index c53679d3..32ecea9e 100644 --- a/src/cdm_data_loaders/validation/xsd/traversal.py +++ b/src/cdm_data_loaders/validation/xsd/traversal.py @@ -1,12 +1,15 @@ """XSD schema loading, multi-level element resolution, and complex-type traversal helpers.""" from collections.abc import Mapping, Sequence -from dataclasses import dataclass, field, replace +from dataclasses import field, replace from pathlib import Path -from typing import Any, Literal, NoReturn +from typing import Annotated, Any, Literal, NoReturn import xmlschema from frozendict import frozendict +from pydantic import AfterValidator, ConfigDict, GetPydanticSchema, InstanceOf, StrictInt, StrictStr +from pydantic.dataclasses import dataclass +from pydantic_core import core_schema from xmlschema.validators import ( XsdAttribute, XsdComplexType, @@ -14,7 +17,7 @@ XsdType, ) -from cdm_data_loaders.utils.recursion import validate_max_recursion_depth +from cdm_data_loaders.utils.recursion import StrictPositiveRecursionDepth, validate_max_recursion_depth from cdm_data_loaders.utils.xml_utils import local_name as _local_name from cdm_data_loaders.validation.xsd.particles import ( group_max_occurs_by_name, @@ -41,16 +44,37 @@ def load_schema(schema_path: str | Path) -> xmlschema.XMLSchema: return xmlschema.XMLSchema(str(schema_path)) -@dataclass(frozen=True, slots=True) +def _freeze_exception_map( + mapping: Mapping[str, type[Exception]], +) -> frozendict[str, type[Exception]]: + """Copy a validated exception mapping into an immutable ``frozendict``. + + :param mapping: Mapping from specification code to exception class. + :type mapping: Mapping[str, type[Exception]] + :return: Immutable ``frozendict`` copy. + :rtype: frozendict[str, type[Exception]] + """ + return frozendict(mapping) + + +type _StructCacheField = Annotated[ + dict[tuple[int, bool], Any], + GetPydanticSchema(lambda _tp, _handler: core_schema.is_instance_schema(dict)), +] + + +@dataclass(frozen=True, slots=True, config=ConfigDict(arbitrary_types_allowed=True, extra="forbid")) class TraversalContext: """Immutable traversal context for XSD schema lowering and XML-to-dict coercion.""" - schema: xmlschema.XMLSchema - type_counts: Mapping[int, int] - max_recursion_depth: int | None - struct_cache: dict[tuple[int, bool], Any] = field(default_factory=dict) - error_type: type[Exception] = ValueError - error_by_code: Mapping[str, type[Exception]] = field(default_factory=frozendict) + schema: InstanceOf[xmlschema.XMLSchema] + type_counts: Mapping[StrictInt, StrictInt] + max_recursion_depth: StrictPositiveRecursionDepth | None + struct_cache: _StructCacheField = field(default_factory=dict) + error_type: type[Exception] = field(default_factory=lambda: ValueError) + error_by_code: Annotated[Mapping[StrictStr, type[Exception]], AfterValidator(_freeze_exception_map)] = field( + default_factory=frozendict + ) def raise_err(self, code: str, msg: str) -> NoReturn: """Raise the configured exception subclass for ``code`` or fall back to ``error_type``. diff --git a/tests/cdm_data_loaders/cli/test_ncbi_ftp_filter_updates.py b/tests/cdm_data_loaders/cli/test_ncbi_ftp_filter_updates.py index e451b562..ade40d4b 100644 --- a/tests/cdm_data_loaders/cli/test_ncbi_ftp_filter_updates.py +++ b/tests/cdm_data_loaders/cli/test_ncbi_ftp_filter_updates.py @@ -65,8 +65,31 @@ def test_settings_date_formats_and_paths( assert settings.mod_date == expected_date assert settings.mod_date_tag == "20240501" assert settings.derived_dir == out_dir / "derived" - assert settings.source_dir(Source.REFSEQ) == out_dir / "refseq" - assert settings.source_dir(Source.GENBANK) == out_dir / "genbank" + + +@pytest.mark.parametrize( + ("source", "expected_subdir"), + [ + pytest.param(Source.REFSEQ, "refseq", id="refseq"), + pytest.param(Source.GENBANK, "genbank", id="genbank"), + ], +) +def test_settings_source_dir( + input_files: tuple[Path, Path], + tmp_path: Path, + source: Source, + expected_subdir: str, +) -> None: + """Resolve per-source output subdirectory under output_dir.""" + mod_times, interesting = input_files + out_dir = tmp_path / "out" + settings = FilterNcbiUpdatesSettings( + ftp_mod_times_path=mod_times, + assemblies_of_interest_path=interesting, + mod_date="2024/05/01", + output_dir=out_dir, + ) + assert settings.source_dir(source) == out_dir / expected_subdir @pytest.mark.parametrize( diff --git a/tests/cdm_data_loaders/coercion/test_arrow_core.py b/tests/cdm_data_loaders/coercion/test_arrow_core.py index 7fc9ab8a..dbebbbfd 100644 --- a/tests/cdm_data_loaders/coercion/test_arrow_core.py +++ b/tests/cdm_data_loaders/coercion/test_arrow_core.py @@ -3,6 +3,7 @@ from datetime import UTC, date, datetime, time, timedelta, timezone from decimal import Decimal from enum import StrEnum +from typing import Final import pyarrow as pa import pytest @@ -212,28 +213,51 @@ def test_coerce_numeric_converts_and_preserves_guards( assert coerce_numeric(raw, target_type) == expected -def test_coerce_string_and_enum_unwrapping() -> None: - """Unwrap Enum values and format temporal, binary, and container values for string columns.""" +def test_coerce_scalar_for_pyarrow_unwraps_enum() -> None: + """Unwrap StrEnum values for PyArrow string columns.""" assert coerce_scalar_for_pyarrow(_ColorEnum.RED, pa.string()) == "red" - assert coerce_string(UTC_REF_DT) == "2024-02-29T12:30:00+00:00" - assert coerce_string(b"text") == "text" - assert coerce_string((1, date(2024, 1, 1))) == '[1, "2024-01-01"]' -def test_coerce_struct_from_mapping_or_tuple_handles_tuple_union_and_passthrough() -> None: +@pytest.mark.parametrize( + ("raw", "expected"), + [ + pytest.param(UTC_REF_DT, "2024-02-29T12:30:00+00:00", id="utc-datetime"), + pytest.param(b"text", "text", id="bytes"), + pytest.param((1, date(2024, 1, 1)), '[1, "2024-01-01"]', id="tuple-with-date"), + ], +) +def test_coerce_string_formats_values(raw: object, expected: str) -> None: + """Format temporal, binary, and container values for string columns.""" + assert coerce_string(raw) == expected + + +TUPLE_STRUCT: Final[pa.StructType] = pa.struct([pa.field("item_0", pa.int64()), pa.field("item_1", pa.string())]) +UNION_STRUCT: Final[pa.StructType] = pa.struct([pa.field("label", pa.string()), pa.field("_value", pa.int64())]) +PLAIN_STRUCT: Final[pa.StructType] = pa.struct([pa.field("label", pa.string())]) + + +@pytest.mark.parametrize( + ("raw", "struct_type", "allow_tuple", "expected"), + [ + pytest.param((1, "a"), TUPLE_STRUCT, False, (1, "a"), id="tuple-disallowed-passthrough"), + pytest.param((1, None), TUPLE_STRUCT, True, {"item_0": 1}, id="tuple-allowed-omits-none"), + pytest.param("42", UNION_STRUCT, False, {"_value": 42}, id="scalar-union-value-field"), + pytest.param("orphan", PLAIN_STRUCT, False, "orphan", id="non-mapping-passthrough"), + ], +) +def test_coerce_struct_from_mapping_or_tuple_handles_tuple_union_and_passthrough( + raw: object, + struct_type: pa.StructType, + allow_tuple: bool, + expected: object, +) -> None: """Coerce positional tuples, _value scalar unions, and non-matching values on StructType.""" - tuple_struct = pa.struct([pa.field("item_0", pa.int64()), pa.field("item_1", pa.string())]) - union_struct = pa.struct([pa.field("label", pa.string()), pa.field("_value", pa.int64())]) - plain_struct = pa.struct([pa.field("label", pa.string())]) - - assert coerce_struct_from_mapping_or_tuple( - (1, "a"), tuple_struct, coerce_scalar_for_pyarrow, allow_tuple=False - ) == ( - 1, - "a", + assert ( + coerce_struct_from_mapping_or_tuple( + raw, + struct_type, + coerce_scalar_for_pyarrow, + allow_tuple=allow_tuple, + ) + == expected ) - assert coerce_struct_from_mapping_or_tuple( - (1, None), tuple_struct, coerce_scalar_for_pyarrow, allow_tuple=True - ) == {"item_0": 1} - assert coerce_struct_from_mapping_or_tuple("42", union_struct, coerce_scalar_for_pyarrow) == {"_value": 42} - assert coerce_struct_from_mapping_or_tuple("orphan", plain_struct, coerce_scalar_for_pyarrow) == "orphan" diff --git a/tests/cdm_data_loaders/coercion/test_pydantic_to_arrow.py b/tests/cdm_data_loaders/coercion/test_pydantic_to_arrow.py index 562988a5..3974ff32 100644 --- a/tests/cdm_data_loaders/coercion/test_pydantic_to_arrow.py +++ b/tests/cdm_data_loaders/coercion/test_pydantic_to_arrow.py @@ -31,10 +31,17 @@ class _SampleRecord(BaseModel): created_at: datetime -def test_coerce_pydantic_record_for_pyarrow_pass_model_and_mapping_parity() -> None: +@pytest.mark.parametrize( + "from_model", + [ + pytest.param(True, id="validated-model"), + pytest.param(False, id="raw-mapping"), + ], +) +def test_coerce_pydantic_record_for_pyarrow_pass_model_and_mapping_parity(from_model: bool) -> None: """coerce_pydantic_record_for_pyarrow produces identical rows from validated models and raw dicts.""" pa_schema = generate_pyarrow_schema(_SampleRecord, by_alias=True) - raw = { + raw: dict[str, Any] = { "record_id": "rec-1", "secret_token": "top-secret", "raw_key": b"\x01\x02", @@ -43,13 +50,11 @@ def test_coerce_pydantic_record_for_pyarrow_pass_model_and_mapping_parity() -> N "duration": "PT90S", "created_at": "2024-05-01T10:00:00Z", } - instance = _SampleRecord.model_validate(raw) + record_input: _SampleRecord | dict[str, Any] = _SampleRecord.model_validate(raw) if from_model else raw + model_arg = None if from_model else _SampleRecord - from_model = coerce_pydantic_record_for_pyarrow(instance, pa_schema, by_alias=True) - from_dict = coerce_pydantic_record_for_pyarrow(raw, pa_schema, model=_SampleRecord, by_alias=True) - - assert from_model == from_dict - assert from_model == { + coerced = coerce_pydantic_record_for_pyarrow(record_input, pa_schema, model=model_arg, by_alias=True) + assert coerced == { "recordId": "rec-1", "secret_token": "top-secret", "raw_key": b"\x01\x02", @@ -58,7 +63,7 @@ def test_coerce_pydantic_record_for_pyarrow_pass_model_and_mapping_parity() -> N "duration": "PT1M30S", "created_at": datetime(2024, 5, 1, 10, 0, tzinfo=UTC), } - table = pa.Table.from_pylist([from_model], schema=pa_schema) + table = pa.Table.from_pylist([coerced], schema=pa_schema) assert table.num_rows == 1 @@ -187,28 +192,47 @@ class _ScalarRootModel(RootModel[int]): pass -def test_coerce_pydantic_record_for_pyarrow_pass_root_models_excluded_fields_and_dataclass() -> None: +@pytest.mark.parametrize( + ("instance", "pa_schema", "expected"), + [ + pytest.param( + _NestedRootModel(_ModelWithExcludedAndComputed(visible=5, coord=_Coord(x=3))), + pa.schema( + [ + pa.field("visible", pa.int64()), + pa.field("hidden", pa.string()), + pa.field("doubled", pa.int64()), + pa.field("coord", pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())])), + ] + ), + { + "visible": 5, + "doubled": 10, + "coord": {"x": 3}, + }, + id="nested-root-model", + ), + pytest.param( + _MappingRootModel({"k": 9}), + pa.schema([pa.field("k", pa.int64())]), + {"k": 9}, + id="mapping-root-model", + ), + pytest.param( + _ScalarRootModel(42), + pa.schema([pa.field("_value", pa.int64())]), + {"_value": 42}, + id="scalar-root-model", + ), + ], +) +def test_coerce_pydantic_record_for_pyarrow_pass_root_models_excluded_fields_and_dataclass( + instance: BaseModel, + pa_schema: pa.Schema, + expected: dict[str, Any], +) -> None: """RootModel variants, excluded fields, computed fields, and dataclass structs coerce as expected.""" - nested_schema = pa.schema( - [ - pa.field("visible", pa.int64()), - pa.field("hidden", pa.string()), - pa.field("doubled", pa.int64()), - pa.field("coord", pa.struct([pa.field("x", pa.int64()), pa.field("y", pa.int64())])), - ] - ) - nested_instance = _NestedRootModel(_ModelWithExcludedAndComputed(visible=5, coord=_Coord(x=3))) - assert coerce_pydantic_record_for_pyarrow(nested_instance, nested_schema) == { - "visible": 5, - "doubled": 10, - "coord": {"x": 3}, - } - - map_schema = pa.schema([pa.field("k", pa.int64())]) - assert coerce_pydantic_record_for_pyarrow(_MappingRootModel({"k": 9}), map_schema) == {"k": 9} - - scalar_schema = pa.schema([pa.field("_value", pa.int64())]) - assert coerce_pydantic_record_for_pyarrow(_ScalarRootModel(42), scalar_schema) == {"_value": 42} + assert coerce_pydantic_record_for_pyarrow(instance, pa_schema) == expected def test_to_jsonable_recursive_pass_root_model_enum_network_and_scalars() -> None: diff --git a/tests/cdm_data_loaders/converters/core/test_dlt_normalization.py b/tests/cdm_data_loaders/converters/core/test_dlt_normalization.py index b9a6eef9..d8035c6c 100644 --- a/tests/cdm_data_loaders/converters/core/test_dlt_normalization.py +++ b/tests/cdm_data_loaders/converters/core/test_dlt_normalization.py @@ -62,37 +62,53 @@ def test_child_table_name_pass_literal_join(parent: str, key: str, expected: str assert child_table_name(parent, key) == expected -def test_unflatten_tables_pass_precise_forest_roundtrip() -> None: +@pytest.mark.parametrize( + "parents_first_input", + [ + pytest.param(False, id="children-first-order"), + pytest.param(True, id="parents-first-order"), + ], +) +def test_unflatten_tables_pass_precise_forest_roundtrip(parents_first_input: bool) -> None: """Preserve every raw field while following explicit parents in declaration order.""" - tables = json.loads(Path("tests/data/dlt_normalization/tables.json").read_text(encoding="utf-8")) + raw_tables = json.loads(Path("tests/data/dlt_normalization/tables.json").read_text(encoding="utf-8")) + expected_order = [ + "catalog__literal", + "root", + "root__root__tags", + "unrelated", + "unrelated__leaf", + "root__looks_nested", + ] + tables = {name: raw_tables[name] for name in expected_order} if parents_first_input else raw_tables original = deepcopy(tables) roots = unflatten_tables(tables) - leaf = DltTableNode("unrelated__leaf", "unrelated", "leaf", tables["unrelated__leaf"]) - unrelated = DltTableNode("unrelated", "root", "unrelated", tables["unrelated"], {"leaf": leaf}) - tags = DltTableNode("root__root__tags", "root", "root__tags", tables["root__root__tags"]) + leaf = DltTableNode(name="unrelated__leaf", parent="unrelated", key="leaf", table=tables["unrelated__leaf"]) + unrelated = DltTableNode( + name="unrelated", parent="root", key="unrelated", table=tables["unrelated"], children={"leaf": leaf} + ) + tags = DltTableNode(name="root__root__tags", parent="root", key="root__tags", table=tables["root__root__tags"]) assert roots == { - "catalog__literal": DltTableNode("catalog__literal", None, "catalog__literal", tables["catalog__literal"]), - "root": DltTableNode("root", None, "root", tables["root"], {"root__tags": tags, "unrelated": unrelated}), + "catalog__literal": DltTableNode( + name="catalog__literal", parent=None, key="catalog__literal", table=tables["catalog__literal"] + ), + "root": DltTableNode( + name="root", + parent=None, + key="root", + table=tables["root"], + children={"root__tags": tags, "unrelated": unrelated}, + ), "root__looks_nested": DltTableNode( - "root__looks_nested", None, "root__looks_nested", tables["root__looks_nested"] + name="root__looks_nested", parent=None, key="root__looks_nested", table=tables["root__looks_nested"] ), } assert list(roots) == ["catalog__literal", "root", "root__looks_nested"] assert list(roots["root"].children) == ["root__tags", "unrelated"] flattened = flatten_nodes(roots) - assert flattened == original - expected_order = [ - "catalog__literal", - "root", - "root__root__tags", - "unrelated", - "unrelated__leaf", - "root__looks_nested", - ] + assert flattened == {name: original[name] for name in expected_order} assert list(flattened) == expected_order assert tables_parents_first(tables) == [original[name] for name in expected_order] - assert unflatten_tables(flattened) == roots - assert flatten_nodes(unflatten_tables(tables)) == flattened assert tables == original @@ -215,10 +231,14 @@ def test_unflatten_tables_pass_deep_graph(depth: int) -> None: @pytest.mark.parametrize( ("node", "mapping_key", "message"), [ - (DltTableNode("root", None, "root", {"parent": "other"}), "root", "parent does not match"), - (DltTableNode("root", "other", "root", {}), "root", "parent does not match"), - (DltTableNode("root", None, "wrong", {}), "root", "key does not match"), - (DltTableNode("root", None, "root", {}), "wrong", "key does not match"), + ( + DltTableNode(name="root", parent=None, key="root", table={"parent": "other"}), + "root", + "parent does not match", + ), + (DltTableNode(name="root", parent="other", key="root", table={}), "root", "parent does not match"), + (DltTableNode(name="root", parent=None, key="wrong", table={}), "root", "key does not match"), + (DltTableNode(name="root", parent=None, key="root", table={}), "wrong", "key does not match"), ], ids=["raw-parent-mismatch", "node-parent-mismatch", "node-key-mismatch", "mapping-key-mismatch"], ) @@ -231,7 +251,9 @@ def test_flatten_nodes_fail_inconsistent_tree(node: DltTableNode, mapping_key: s @pytest.mark.parametrize("reuse_node", [True, False], ids=["cyclic-node", "duplicate-name"]) def test_flatten_nodes_fail_repeated_table(reuse_node: bool) -> None: """Reject cycles and duplicate names in manually constructed trees.""" - root = DltTableNode("root", None, "root", {}) - root.children["root"] = root if reuse_node else DltTableNode("root", "root", "root", {"parent": "root"}) + root = DltTableNode(name="root", parent=None, key="root", table={}) + root.children["root"] = ( + root if reuse_node else DltTableNode(name="root", parent="root", key="root", table={"parent": "root"}) + ) with pytest.raises(DltNormalizationError, match="Cycle or duplicate table name"): flatten_nodes({"root": root}) diff --git a/tests/cdm_data_loaders/converters/core/test_extensions.py b/tests/cdm_data_loaders/converters/core/test_extensions.py index 61953ca8..aa006128 100644 --- a/tests/cdm_data_loaders/converters/core/test_extensions.py +++ b/tests/cdm_data_loaders/converters/core/test_extensions.py @@ -7,6 +7,7 @@ import pytest from jsonschema.exceptions import SchemaError, ValidationError +from pydantic import ValidationError as PydanticValidationError from cdm_data_loaders.converters.core.extensions import ( DEFAULT_EXTENSIONS, @@ -50,10 +51,12 @@ def test_extensions_pass_decimal_multiple_of_float( divisor: float | Decimal, ) -> None: """Fractional validation accepts exact multiples regardless of Decimal context precision.""" - registry = ExtensionRegistry((ExtensionSpec("x-amount", {"type": "number", "multipleOf": divisor}),)) + registry = ExtensionRegistry( + specs=(ExtensionSpec(namespace="x-amount", schema={"type": "number", "multipleOf": divisor}),) + ) with localcontext() as context: context.prec = 2 - extensions = Extensions({"x-amount": amount}, registry) + extensions = Extensions(payload={"x-amount": amount}, registry=registry) assert extensions["x-amount"] is amount @@ -83,11 +86,13 @@ def test_extensions_fail_decimal_multiple_of_float( invalid: float | Decimal, ) -> None: """Fractional validation rejects non-multiples regardless of Decimal context precision.""" - registry = ExtensionRegistry((ExtensionSpec("x-amount", {"type": "number", "multipleOf": divisor}),)) + registry = ExtensionRegistry( + specs=(ExtensionSpec(namespace="x-amount", schema={"type": "number", "multipleOf": divisor}),) + ) with localcontext() as context: context.prec = 2 with pytest.raises(ExtensionError, match=r"Invalid x-amount at \[\]: .*not a multiple"): - Extensions({"x-amount": invalid}, registry) + Extensions(payload={"x-amount": invalid}, registry=registry) @pytest.mark.parametrize( @@ -112,9 +117,11 @@ def test_extensions_pass_mixed_numeric_keywords( accepted: str, ) -> None: """All numeric keywords accept valid mixed representations consistently.""" - registry = ExtensionRegistry((ExtensionSpec("x-amount", {"type": "number", keyword: schema_type("2")}),)) + registry = ExtensionRegistry( + specs=(ExtensionSpec(namespace="x-amount", schema={"type": "number", keyword: schema_type("2")}),) + ) amount = instance_type(accepted) - assert Extensions({"x-amount": amount}, registry)["x-amount"] is amount + assert Extensions(payload={"x-amount": amount}, registry=registry)["x-amount"] is amount @pytest.mark.parametrize( @@ -139,9 +146,11 @@ def test_extensions_fail_mixed_numeric_keywords( rejected: str, ) -> None: """All numeric keywords reject invalid mixed representations consistently.""" - registry = ExtensionRegistry((ExtensionSpec("x-amount", {"type": "number", keyword: schema_type("2")}),)) + registry = ExtensionRegistry( + specs=(ExtensionSpec(namespace="x-amount", schema={"type": "number", keyword: schema_type("2")}),) + ) with pytest.raises(ExtensionError) as error: - Extensions({"x-amount": instance_type(rejected)}, registry) + Extensions(payload={"x-amount": instance_type(rejected)}, registry=registry) assert isinstance(error.value.__cause__, ValidationError) assert error.value.__cause__.validator == keyword @@ -160,12 +169,14 @@ def test_extensions_pass_exact_fractional_bounds( amount: float | Decimal, boundary: float | Decimal, keyword: str ) -> None: """Decimal and JSON float boundaries compare as equal without binary rounding.""" - registry = ExtensionRegistry((ExtensionSpec("x-amount", {"type": "number", keyword: boundary}),)) + registry = ExtensionRegistry( + specs=(ExtensionSpec(namespace="x-amount", schema={"type": "number", keyword: boundary}),) + ) if keyword.startswith("exclusive"): with pytest.raises(ExtensionError): - Extensions({"x-amount": amount}, registry) + Extensions(payload={"x-amount": amount}, registry=registry) else: - assert Extensions({"x-amount": amount}, registry)["x-amount"] is amount + assert Extensions(payload={"x-amount": amount}, registry=registry)["x-amount"] is amount def _nested_numeric_ref_registry() -> ExtensionRegistry: @@ -183,14 +194,14 @@ def _nested_numeric_ref_registry() -> ExtensionRegistry: "type": "array", "items": {"type": "object", "properties": {"amount": {"$ref": "#/$defs/amount"}}}, } - return ExtensionRegistry((ExtensionSpec("x-amount", schema),)) + return ExtensionRegistry(specs=(ExtensionSpec(namespace="x-amount", schema=schema),)) def test_extensions_pass_nested_numeric_refs() -> None: """Local refs retain exact arithmetic and preserve Decimal identity in valid nested items.""" registry = _nested_numeric_ref_registry() amount = Decimal("0.3") - extensions = Extensions({"x-amount": [{"amount": amount}]}, registry) + extensions = Extensions(payload={"x-amount": [{"amount": amount}]}, registry=registry) assert extensions["x-amount"] == ({"amount": amount},) assert extensions["x-amount"][0]["amount"] is amount @@ -199,7 +210,7 @@ def test_extensions_fail_nested_numeric_refs() -> None: """Local refs report nested validation error paths when exact arithmetic fails.""" registry = _nested_numeric_ref_registry() with pytest.raises(ExtensionError, match=r"Invalid x-amount at \[0, 'amount'\]: .*not a multiple") as error: - Extensions({"x-amount": [{"amount": Decimal("0.31")}]}, registry) + Extensions(payload={"x-amount": [{"amount": Decimal("0.31")}]}, registry=registry) assert isinstance(error.value.__cause__, ValidationError) assert list(error.value.__cause__.path) == [0, "amount"] @@ -212,12 +223,12 @@ def test_extensions_fail_nested_numeric_refs() -> None: ) def test_extensions_pass_boolean_schemas(value: Value, schema: bool) -> None: """True and false schemas remain unconditional for all finite payloads.""" - registry = ExtensionRegistry((ExtensionSpec("x-value", schema=schema),)) + registry = ExtensionRegistry(specs=(ExtensionSpec(namespace="x-value", schema=schema),)) if schema: - assert Extensions({"x-value": value}, registry)["x-value"] == freeze_value(value) + assert Extensions(payload={"x-value": value}, registry=registry)["x-value"] == freeze_value(value) else: with pytest.raises(ExtensionError, match="False schema"): - Extensions({"x-value": value}, registry) + Extensions(payload={"x-value": value}, registry=registry) @pytest.mark.parametrize( @@ -253,23 +264,23 @@ def test_extensions_pass_boolean_schemas(value: Value, schema: bool) -> None: ) def test_extensions_pass_custom_schema_types(schema: dict[str, Any], value: Value, valid: bool) -> None: """Numeric normalization preserves type checks and literal schema comparisons.""" - registry = ExtensionRegistry((ExtensionSpec("x-value", schema),)) + registry = ExtensionRegistry(specs=(ExtensionSpec(namespace="x-value", schema=schema),)) if valid: - assert Extensions({"x-value": value}, registry)["x-value"] == freeze_value(value) + assert Extensions(payload={"x-value": value}, registry=registry)["x-value"] == freeze_value(value) else: with pytest.raises(ExtensionError): - Extensions({"x-value": value}, registry) + Extensions(payload={"x-value": value}, registry=registry) def _spec_with_minimum(value: float | Decimal) -> ExtensionSpec: """Construct an ExtensionSpec with a numeric minimum bound.""" - return ExtensionSpec("x-amount", {"minimum": value}) + return ExtensionSpec(namespace="x-amount", schema={"minimum": value}) def _extensions_with_amount(value: float | Decimal) -> Extensions: """Construct an Extensions payload for an unconditional x-amount namespace.""" - registry = ExtensionRegistry((ExtensionSpec("x-amount", schema=True),)) - return Extensions({"x-amount": value}, registry) + registry = ExtensionRegistry(specs=(ExtensionSpec(namespace="x-amount", schema=True),)) + return Extensions(payload={"x-amount": value}, registry=registry) @pytest.mark.parametrize( @@ -304,7 +315,7 @@ def test_extensions_fail_nonfinite_numbers( ) def test_extensions_pass_builtin(values: dict[str, Any]) -> None: """Builtin namespaces validate exact internal key spellings.""" - extensions = Extensions(values) + extensions = Extensions(payload=values) assert list(extensions) == list(values) assert len(extensions) == len(values) assert TypedNode(type="any", extensions=extensions).extensions is extensions @@ -333,9 +344,9 @@ def test_extensions_fail_invalid(values: dict[str, Any]) -> None: def test_extension_registry_pass_custom_copy_isolation() -> None: """Custom arrays and null are allowed only under an explicit schema.""" schema = {"type": "array", "items": {"type": ["string", "null"]}} - registry = DEFAULT_EXTENSIONS.register(ExtensionSpec("x-custom", schema)) + registry = DEFAULT_EXTENSIONS.register(ExtensionSpec(namespace="x-custom", schema=schema)) values = {"x-custom": ["data", None]} - extensions = Extensions(values, registry) + extensions = Extensions(payload=values, registry=registry) schema["items"]["type"].append("number") values["x-custom"].append("changed") assert extensions["x-custom"] == ("data", None) @@ -353,10 +364,10 @@ def test_extension_registry_pass_custom_copy_isolation() -> None: def test_extension_spec_fail_namespace(namespace: str) -> None: """Only named x-prefixed namespaces can be registered.""" with pytest.raises(ExtensionError): - ExtensionSpec(namespace, schema=True) + ExtensionSpec(namespace=namespace, schema=True) -DUPLICATE_TEST_SPEC: Final[ExtensionSpec] = ExtensionSpec("x-test", schema=True) +DUPLICATE_TEST_SPEC: Final[ExtensionSpec] = ExtensionSpec(namespace="x-test", schema=True) @pytest.mark.parametrize( @@ -368,7 +379,7 @@ def test_extension_spec_fail_namespace(namespace: str) -> None: id="duplicate-in-constructor", ), pytest.param( - partial(ExtensionRegistry((DUPLICATE_TEST_SPEC,)).register, DUPLICATE_TEST_SPEC), + partial(ExtensionRegistry(specs=(DUPLICATE_TEST_SPEC,)).register, DUPLICATE_TEST_SPEC), "Already registered", id="duplicate-on-register", ), @@ -382,7 +393,7 @@ def test_extension_registry_fail_duplicates(call: Callable[[], object], match: s def test_extension_registry_pass_register_replace() -> None: """Explicit replacement updates an existing namespace contract.""" - registry = ExtensionRegistry((DUPLICATE_TEST_SPEC,)) + registry = ExtensionRegistry(specs=(DUPLICATE_TEST_SPEC,)) assert registry.register(DUPLICATE_TEST_SPEC, replace=True) == registry @@ -397,13 +408,14 @@ def test_extension_registry_pass_register_replace() -> None: def test_extension_spec_fail_invalid_schemas(schema: object, error_type: type[Exception], match: str | None) -> None: """Invalid extension validation schemas fail at spec construction.""" with pytest.raises(error_type, match=match): - ExtensionSpec("x-bad", schema) + ExtensionSpec(namespace="x-bad", schema=schema) def test_extension_registry_fail_invalid_entries() -> None: - """Registry entries must be ExtensionSpec instances.""" - with pytest.raises(TypeError, match="entries"): - ExtensionRegistry(({},)) # type: ignore[arg-type] + """Registry entries must be ExtensionSpec instances and registries use __slots__.""" + assert not hasattr(DEFAULT_EXTENSIONS, "__dict__") + with pytest.raises(PydanticValidationError, match="Input should be an instance of ExtensionSpec"): + ExtensionRegistry(specs=({},)) # type: ignore[arg-type] @pytest.mark.parametrize( @@ -425,7 +437,7 @@ def test_extensions_pass_exact_xsv_schema_and_mapping_api() -> None: """The exact existing XSV schema is reused, and mapping methods remain callable.""" xsv_config_schema = get_xsv_config_schema() spec = next(spec for spec in DEFAULT_EXTENSIONS.specs if spec.namespace == "x-xsv-config") - extensions = Extensions({"x-pii": True}) + extensions = Extensions(payload={"x-pii": True}) assert list(extensions.values()) == [True] assert list(extensions.items()) == [("x-pii", True)] assert spec.schema["additionalProperties"] == xsv_config_schema["additionalProperties"] @@ -445,18 +457,18 @@ def load_schema() -> dict[str, Any]: calls += 1 return {"type": "string", "minLength": 1} - spec = ExtensionSpec("x-lazy", load_schema) - registry = ExtensionRegistry((spec, ExtensionSpec("x-other", {"type": "boolean"}))) + spec = ExtensionSpec(namespace="x-lazy", schema=load_schema) + registry = ExtensionRegistry(specs=(spec, ExtensionSpec(namespace="x-other", schema={"type": "boolean"}))) assert calls == 0 - assert Extensions({"x-other": True}, registry)["x-other"] is True + assert Extensions(payload={"x-other": True}, registry=registry)["x-other"] is True assert calls == 0 - assert Extensions({"x-lazy": "ok"}, registry)["x-lazy"] == "ok" + assert Extensions(payload={"x-lazy": "ok"}, registry=registry)["x-lazy"] == "ok" assert calls == 1 assert spec.schema == {"type": "string", "minLength": 1} assert calls == 1 - assert spec == ExtensionSpec("x-lazy", {"type": "string", "minLength": 1}) + assert spec == ExtensionSpec(namespace="x-lazy", schema={"type": "string", "minLength": 1}) assert spec != "x-lazy" - assert hash(spec) == hash(ExtensionSpec("x-lazy", {"type": "string", "minLength": 1})) + assert hash(spec) == hash(ExtensionSpec(namespace="x-lazy", schema={"type": "string", "minLength": 1})) @pytest.mark.parametrize( @@ -468,7 +480,7 @@ def load_schema() -> dict[str, Any]: ) def test_extension_spec_fail_lazy_loader_invalid_schema(bad_schema: object, error_type: type[Exception]) -> None: """Invalid schemas returned by a lazy loader fail when `.schema` is accessed.""" - spec = ExtensionSpec("x-lazy", lambda: bad_schema) # type: ignore[arg-type, return-value] + spec = ExtensionSpec(namespace="x-lazy", schema=lambda: bad_schema) # type: ignore[arg-type, return-value] with pytest.raises(error_type): _ = spec.schema @@ -504,11 +516,11 @@ def test_extension_spec_fail_lazy_loader_invalid_schema(bad_schema: object, erro @pytest.mark.parametrize("hint", DLT_BOOLEAN_HINTS, ids=DLT_BOOLEAN_HINT_IDS) def test_extensions_pass_dlt_boolean_hints(hint: str) -> None: """Each observed dlt column hint accepts boolean values.""" - assert Extensions({"x-dlt": {hint: True}})["x-dlt"] == {hint: True} + assert Extensions(payload={"x-dlt": {hint: True}})["x-dlt"] == {hint: True} @pytest.mark.parametrize("hint", DLT_BOOLEAN_HINTS, ids=DLT_BOOLEAN_HINT_IDS) def test_extensions_fail_dlt_nonboolean_hints(hint: str) -> None: """Each observed dlt column hint rejects non-boolean values.""" with pytest.raises(ExtensionError): - Extensions({"x-dlt": {hint: "yes"}}) + Extensions(payload={"x-dlt": {hint: "yes"}}) diff --git a/tests/cdm_data_loaders/converters/core/test_ir.py b/tests/cdm_data_loaders/converters/core/test_ir.py index 4862fa0d..fa76a326 100644 --- a/tests/cdm_data_loaders/converters/core/test_ir.py +++ b/tests/cdm_data_loaders/converters/core/test_ir.py @@ -1,13 +1,14 @@ """Target-independent schema model contracts.""" from collections.abc import Callable -from dataclasses import FrozenInstanceError +from dataclasses import FrozenInstanceError, replace from decimal import Decimal from functools import partial from typing import Any, Final from unittest.mock import patch import pytest +from pydantic import ValidationError from cdm_data_loaders.converters.core import ir from cdm_data_loaders.converters.core.errors import ConversionError @@ -17,8 +18,8 @@ METADATA_POSITIONS: Final = [ pytest.param(partial(TypedNode, type="any"), "constraints", id="type-constraints"), pytest.param(partial(TypedNode, type="any"), "annotations", id="type-annotations"), - pytest.param(partial(Field, "value", TypedNode(type="any")), "annotations", id="field-annotations"), - pytest.param(partial(SchemaDocument, TypedNode(type="any")), "annotations", id="document-annotations"), + pytest.param(partial(Field, name="value", node=TypedNode(type="any")), "annotations", id="field-annotations"), + pytest.param(partial(SchemaDocument, root=TypedNode(type="any")), "annotations", id="document-annotations"), ] @@ -52,10 +53,12 @@ def test_schema_document_pass_owned_values_and_presence() -> None: """Field presence, nullable values and nested metadata remain independent.""" values = {"enum": [{"amount": Decimal("1.2300")}], "const": None} node = TypedNode(type="number", nullable=True, constraints=values) - document = SchemaDocument(root=TypedNode(type="object", properties=(Field("amount", node, required=True),))) + document = SchemaDocument( + root=TypedNode(type="object", properties=(Field(name="amount", node=node, required=True),)) + ) values["enum"].clear() assert node.constraints == {"enum": ({"amount": Decimal("1.2300")},), "const": None} - assert document.root.properties == (Field("amount", node, required=True),) + assert document.root.properties == (Field(name="amount", node=node, required=True),) with pytest.raises(TypeError): node.constraints["const"] = 1 with pytest.raises(FrozenInstanceError): @@ -63,7 +66,7 @@ def test_schema_document_pass_owned_values_and_presence() -> None: def test_schema_document_pass_post_init_freezes_annotations_once() -> None: - """__post_init__ freezes document annotations exactly once, not once per identity field.""" + """Document construction freezes annotations exactly once, not once per identity field.""" root = TypedNode(type="any") with patch("cdm_data_loaders.converters.core.ir._schema_metadata", wraps=ir._schema_metadata) as spy: # noqa: SLF001 SchemaDocument(root=root, name="n", dialect="d", identifier="i", annotations={"a": 1}) @@ -82,7 +85,7 @@ def test_freeze_value_fail_invalid(value: object) -> None: def test_provenance_pass_owned_metadata() -> None: """Metadata preserves absent versus explicit null values.""" metadata = {"description": None, "tags": ["one"]} - provenance = Provenance("dlt", ("tables", "entries"), metadata) + provenance = Provenance(source="dlt", path=("tables", "entries"), metadata=metadata) metadata["tags"].append("two") assert provenance.metadata == {"description": None, "tags": ("one",)} assert "default" not in provenance.metadata @@ -98,7 +101,7 @@ def test_provenance_pass_owned_metadata() -> None: {"declared_type": [1]}, {"declared_type": 1}, {"properties": [1]}, - {"properties": [Field("a", TypedNode(type="any")), Field("a", TypedNode(type="any"))]}, + {"properties": [Field(name="a", node=TypedNode(type="any")), Field(name="a", node=TypedNode(type="any"))]}, {"required_names": [1]}, {"items": 1}, {"items": [1]}, @@ -133,7 +136,7 @@ def test_provenance_pass_owned_metadata() -> None: ) def test_typed_node_fail_invalid_fields(kwargs: dict[str, Any]) -> None: """Normal constructors reject malformed types and mutable model substitutes.""" - with pytest.raises((TypeError, ValueError)): + with pytest.raises(ValidationError): TypedNode(**{"type": "any", **kwargs}) @@ -149,18 +152,18 @@ def test_typed_node_fail_invalid_fields(kwargs: dict[str, Any]) -> None: ) def test_node_hints_fail_invalid(kwargs: dict[str, Any]) -> None: """Logical hints validate values rather than coercing invalid inputs.""" - with pytest.raises((TypeError, ValueError)): + with pytest.raises(ValidationError): NodeHints(**kwargs) @pytest.mark.parametrize( ("call", "error_type", "match"), [ - (partial(Provenance, "bogus"), ValueError, "source"), - (partial(Provenance, "dlt", (True,)), TypeError, "paths"), - (partial(Field, "name", {}), TypeError, "Fields require"), - (partial(SchemaDocument, root={}), TypeError, "root"), - (partial(SchemaDocument, root=TypedNode(type="any"), name=1), TypeError, "name"), + (partial(Provenance, source="bogus"), ValidationError, "source"), + (partial(Provenance, source="dlt", path=(True,)), ValidationError, "path"), + (partial(Field, name="name", node={}), ValidationError, "node"), + (partial(SchemaDocument, root={}), ValidationError, "root"), + (partial(SchemaDocument, root=TypedNode(type="any"), name=1), ValidationError, "name"), (partial(freeze_mapping, []), TypeError, "mapping"), ], ids=[ @@ -209,3 +212,20 @@ def test_typed_node_pass_all_containers_owned() -> None: value["default"].clear() assert node.annotations["default"] == (1, {"value": Decimal("1.00")}) assert (freeze_value(1.5),) == (1.5,) + + +def test_ir_models_pass_slots_and_replace_revalidation() -> None: + """All Converter IR dataclasses enforce __slots__ and revalidate on dataclasses.replace.""" + prov = Provenance("dlt", ("tables", 0)) + hints = NodeHints("string") + node = TypedNode("string", nullable=True, hints=hints, provenance=prov) + prop = Field("id", node, required=True, provenance=prov) + doc = SchemaDocument(TypedNode("object", properties=(prop,)), "demo") + for instance in (prov, hints, node, prop, doc): + assert not hasattr(instance, "__dict__") + updated = replace(node, nullable=False) + assert updated.nullable is False + with pytest.raises(ValidationError): + replace(node, nullable="bad") # type: ignore[arg-type] + with pytest.raises(ConversionError, match="extensions"): + replace(node, annotations={"x-pii": True}) diff --git a/tests/cdm_data_loaders/converters/emitters/test_json_schema.py b/tests/cdm_data_loaders/converters/emitters/test_json_schema.py index 45a798de..4822830b 100644 --- a/tests/cdm_data_loaders/converters/emitters/test_json_schema.py +++ b/tests/cdm_data_loaders/converters/emitters/test_json_schema.py @@ -8,6 +8,7 @@ from typing import Any, Final, Literal, cast import pytest +from pydantic import ValidationError from pyiceberg.schema import Schema from pyiceberg.types import ( BinaryType, @@ -469,18 +470,18 @@ def test_emit_pass_json_structural_roundtrip(schema: dict[str, Any], dialect: st @pytest.mark.parametrize( ("node", "expected"), [ - (TypedNode("any"), {}), - (TypedNode("never"), {"not": {}}), - (TypedNode("null"), {"type": "null"}), - (TypedNode("boolean", nullable=True), {"anyOf": [{"type": "boolean"}, {"type": "null"}]}), + (TypedNode(type="any"), {}), + (TypedNode(type="never"), {"not": {}}), + (TypedNode(type="null"), {"type": "null"}), + (TypedNode(type="boolean", nullable=True), {"anyOf": [{"type": "boolean"}, {"type": "null"}]}), ( - TypedNode("object", properties=(Field("flag", TypedNode("boolean"), required=True),)), + TypedNode(type="object", properties=(Field(name="flag", node=TypedNode(type="boolean"), required=True),)), {"type": "object", "properties": {"flag": {"type": "boolean"}}, "required": ["flag"]}, ), - (TypedNode("array", items=TypedNode("any")), {"type": "array", "items": {}}), - (TypedNode("any", annotations={"description": "unrestricted"}), {"description": "unrestricted"}), + (TypedNode(type="array", items=TypedNode(type="any")), {"type": "array", "items": {}}), + (TypedNode(type="any", annotations={"description": "unrestricted"}), {"description": "unrestricted"}), ( - TypedNode("map", key_type=TypedNode("integer"), value_type=TypedNode("any")), + TypedNode(type="map", key_type=TypedNode(type="integer"), value_type=TypedNode(type="any")), { "type": "array", "items": { @@ -496,7 +497,7 @@ def test_emit_pass_json_structural_roundtrip(schema: dict[str, Any], dialect: st ) def test_emit_pass_source_neutral_nodes(node: TypedNode, expected: dict[str, Any]) -> None: """Direct typed nodes emit structural schemas without a fabricated source dialect.""" - assert JsonSchemaEmitter().emit(SchemaDocument(node)) == expected + assert JsonSchemaEmitter().emit(SchemaDocument(root=node)) == expected @pytest.mark.parametrize("mode", ["object", "array"], ids=["objects", "arrays"]) @@ -741,7 +742,7 @@ def test_emit_pass_registered_dlt_extensions(preserve: bool) -> None: def test_emit_pass_json_registered_extensions_and_field_scopes() -> None: """Explicit extension contracts preserve nested literal data and field annotations.""" - registry = DEFAULT_EXTENSIONS.register(ExtensionSpec("x-custom", {"type": ["array", "null"]})) + registry = DEFAULT_EXTENSIONS.register(ExtensionSpec(namespace="x-custom", schema={"type": ["array", "null"]})) source = {"$schema": JSON_SCHEMA_DIALECT, "x-custom": [{"$ref": "literal"}], "properties": {"value": False}} document = JsonSchemaReader(extension_registry=registry).read(source) assert JsonSchemaEmitter().emit(document) == source @@ -768,7 +769,7 @@ def test_emit_pass_json_registered_extensions_and_field_scopes() -> None: def test_emit_fail_invalid_extension_contract(payload: dict[str, Any]) -> None: """Malformed extension payloads fail at the IR boundary without silent registration.""" with pytest.raises(ExtensionError): - JsonSchemaEmitter().emit(SchemaDocument(TypedNode("any", extensions=payload))) + JsonSchemaEmitter().emit(SchemaDocument(TypedNode(type="any", extensions=payload))) def test_emit_fail_unknown_dlt_type() -> None: @@ -792,12 +793,14 @@ def test_emit_fail_unsupported_iceberg_types(field_type: IcebergType) -> None: @pytest.mark.parametrize( "node", [ - TypedNode("unknown"), - TypedNode("map"), - TypedNode("map", provenance=Provenance("iceberg")), - TypedNode("number", hints=NodeHints(logical_type="decimal"), provenance=Provenance("iceberg")), - TypedNode("string", hints=NodeHints(logical_type="fixed"), provenance=Provenance("iceberg")), - TypedNode("unknown", hints=NodeHints(logical_type="timestamp_ns"), provenance=Provenance("iceberg")), + TypedNode(type="unknown"), + TypedNode(type="map"), + TypedNode(type="map", provenance=Provenance(source="iceberg")), + TypedNode(type="number", hints=NodeHints(logical_type="decimal"), provenance=Provenance(source="iceberg")), + TypedNode(type="string", hints=NodeHints(logical_type="fixed"), provenance=Provenance(source="iceberg")), + TypedNode( + type="unknown", hints=NodeHints(logical_type="timestamp_ns"), provenance=Provenance(source="iceberg") + ), ], ids=[ "unknown", @@ -811,13 +814,13 @@ def test_emit_fail_unsupported_iceberg_types(field_type: IcebergType) -> None: def test_emit_fail_incomplete_type_facts(node: TypedNode) -> None: """Missing required logical facts and unknown types are not silently guessed.""" with pytest.raises(ConversionError): - JsonSchemaEmitter().emit(SchemaDocument(node)) + JsonSchemaEmitter().emit(SchemaDocument(root=node)) @pytest.mark.parametrize("policy", [None, 0, "true"], ids=["none", "integer", "string"]) def test_json_schema_emitter_fail_nonboolean_policy(policy: object) -> None: """The hint policy accepts actual booleans only.""" - with pytest.raises(TypeError, match="must be a boolean"): + with pytest.raises(ValidationError, match="preserve_unknown_hints"): JsonSchemaEmitter(preserve_unknown_hints=cast("bool", policy)) @@ -866,19 +869,23 @@ def test_emit_pass_sibling_and_call_isolation(logical: str) -> None: ("document", "expected"), [ ( - SchemaDocument(TypedNode("any"), name="records"), + SchemaDocument(root=TypedNode(type="any"), name="records"), {"title": "records"}, ), ( SchemaDocument( TypedNode( - "object", + type="object", properties=( - Field("flag", TypedNode("boolean", hints=NodeHints(logical_type="bool")), required=True), + Field( + name="flag", + node=TypedNode(type="boolean", hints=NodeHints(logical_type="bool")), + required=True, + ), ), ), name="records", - provenance=Provenance("dlt"), + provenance=Provenance(source="dlt"), ), { "type": "object", @@ -893,11 +900,11 @@ def test_emit_pass_sibling_and_call_isolation(logical: str) -> None: ( SchemaDocument( TypedNode( - "integer", + type="integer", hints=NodeHints(logical_type="int", bit_width=32), nullable=True, extensions={"x-iceberg": {"logical_type": "int"}}, - provenance=Provenance("iceberg"), + provenance=Provenance(source="iceberg"), ), dialect="urn:dialect", identifier="urn:identifier", @@ -942,7 +949,7 @@ def test_emit_pass_document_and_type_metadata(document: SchemaDocument, expected pytest.param( True, {}, - (Field("flag", TypedNode("boolean")),), + (Field(name="flag", node=TypedNode(type="boolean")),), {"properties": {"flag": {"type": "boolean"}}}, id="true-with-properties", ), @@ -961,4 +968,4 @@ def test_emit_pass_boolean_node_metadata_and_edits( node = replace(node, annotations=annotations) if properties: node = replace(node, properties=properties) - assert JsonSchemaEmitter().emit(SchemaDocument(node)) == expected + assert JsonSchemaEmitter().emit(SchemaDocument(root=node)) == expected diff --git a/tests/cdm_data_loaders/converters/emitters/test_pyspark.py b/tests/cdm_data_loaders/converters/emitters/test_pyspark.py index 65ad0365..ac21a5d0 100644 --- a/tests/cdm_data_loaders/converters/emitters/test_pyspark.py +++ b/tests/cdm_data_loaders/converters/emitters/test_pyspark.py @@ -98,7 +98,10 @@ def test_emit_pass_root_union_and_inference(declaration: list[str] | None) -> No @pytest.mark.parametrize("validator", [Draft7Validator, Draft202012Validator], ids=["draft7", "draft2020"]) def test_conversion_context_pass_keyword_policy(validator: type) -> None: """Metadata helpers exclude structural keywords and preserve recognized assertions and annotations.""" - context = ConversionContext(validator, frozenset({"pattern", "x-custom", "type", "$id", "typo"})) + context = ConversionContext( + validator_cls=validator, + extra_metadata_keywords=frozenset({"pattern", "x-custom", "type", "$id", "typo"}), + ) known = get_known_jsonschema_keywords(validator) assert {"title", "description", "pattern"} <= known assert not known & REF_AND_IDENTITY_KEYWORDS @@ -139,14 +142,15 @@ def test_emit_pass_invalid_metadata_warns_once(dialect: str, caplog: pytest.LogC def test_build_metadata_pass_annotation_presence(annotations: dict[str, Any], expected: dict[str, Any]) -> None: """Absent, null and empty annotations remain distinct.""" node = TypedNode(type="string", annotations=annotations) - assert PySparkEmitter().build_metadata(node, ConversionContext(Draft202012Validator)) == expected + assert PySparkEmitter().build_metadata(node, ConversionContext(validator_cls=Draft202012Validator)) == expected def test_emit_pass_registered_extensions_and_metadata_isolation() -> None: """Registered extension payloads and exact decimals are copied without sharing mutable output.""" registry = DEFAULT_EXTENSIONS.register( ExtensionSpec( - "x-custom", {"type": "object", "properties": {"values": {"type": "array"}}, "additionalProperties": False} + namespace="x-custom", + schema={"type": "object", "properties": {"values": {"type": "array"}}, "additionalProperties": False}, ) ) schema = { @@ -318,7 +322,7 @@ def test_emit_node_pass_explicit_context_and_branch_metadata() -> None: document = _document( {"title": "Outer", "oneOf": [{"type": "integer", "title": "Inner", "description": "Branch"}], "default": 1} ) - context = ConversionContext(Draft7Validator, frozenset({"default"})) + context = ConversionContext(validator_cls=Draft7Validator, extra_metadata_keywords=frozenset({"default"})) actual = PySparkEmitter().emit_node(document.root, context) assert actual.jsonValue() == { "type": "struct", diff --git a/tests/cdm_data_loaders/converters/facades/test_facades_end_to_end.py b/tests/cdm_data_loaders/converters/facades/test_facades_end_to_end.py index fffb132b..22914d5b 100644 --- a/tests/cdm_data_loaders/converters/facades/test_facades_end_to_end.py +++ b/tests/cdm_data_loaders/converters/facades/test_facades_end_to_end.py @@ -257,7 +257,7 @@ def _build_vendor_converter( "required": ["classification"], "additionalProperties": False, } - registry = DEFAULT_EXTENSIONS.register(ExtensionSpec("x-vendor", contract)) + registry = DEFAULT_EXTENSIONS.register(ExtensionSpec(namespace="x-vendor", schema=contract)) contract["properties"].clear() if route == "dlt": return registry, JSONSchemaToDlt(schema_name="records", extension_registry=registry) @@ -600,22 +600,22 @@ def test_facades_end_to_end_pass_iceberg_fragments(required: bool, default: Deci [ ( JsonSchemaReader().read_node(source=True), - Field("value", JsonSchemaReader().read_node(source=True)), + Field(name="value", node=JsonSchemaReader().read_node(source=True)), True, ), ( JsonSchemaReader().read_node(source=False), - Field("value", JsonSchemaReader().read_node(source=False)), + Field(name="value", node=JsonSchemaReader().read_node(source=False)), False, ), ( JsonSchemaReader().read_node({"type": "integer"}), - Field("value", JsonSchemaReader().read_node({"type": "integer"})), + Field(name="value", node=JsonSchemaReader().read_node({"type": "integer"})), {"type": "integer"}, ), ( TypedNode(type="string"), - Field("value", TypedNode(type="string")), + Field(name="value", node=TypedNode(type="string")), {"type": "string"}, ), ( diff --git a/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py b/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py index 24dd613c..d623143f 100644 --- a/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py +++ b/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py @@ -175,33 +175,30 @@ def test_jsonschema_to_pyarrow_pass_options_reach_emitter() -> None: @pytest.mark.parametrize( - ("constructor", "is_strict_reader"), + "constructor", [ - pytest.param(StrictJsonSchemaReader, True, id="strict-reader"), - pytest.param(JSONSchemaToPyArrow, False, id="jsonschema-to-pyarrow"), - pytest.param(PydanticToPyArrow, False, id="pydantic-to-pyarrow"), - pytest.param(functools.partial(XsdToPyArrow, xml_tag="entry"), False, id="xsd-to-pyarrow"), + pytest.param(StrictJsonSchemaReader, id="strict-reader"), + pytest.param(JSONSchemaToPyArrow, id="jsonschema-to-pyarrow"), + pytest.param(PydanticToPyArrow, id="pydantic-to-pyarrow"), + pytest.param(functools.partial(XsdToPyArrow, xml_tag="entry"), id="xsd-to-pyarrow"), ], ) @pytest.mark.parametrize( - ("bad_depth", "reader_exc"), + "bad_depth", [ - pytest.param(0, ValueError, id="zero"), - pytest.param(-1, ValueError, id="negative"), - pytest.param(True, TypeError, id="bool-true"), - pytest.param(False, TypeError, id="bool-false"), + pytest.param(0, id="zero"), + pytest.param(-1, id="negative"), + pytest.param(True, id="bool-true"), + pytest.param(False, id="bool-false"), ], ) def test_converters_and_strict_reader_fail_invalid_max_recursion_depth( constructor: Callable[..., object], *, - is_strict_reader: bool, bad_depth: int, - reader_exc: type[Exception], ) -> None: """Reject non-positive or boolean max_recursion_depth across StrictJsonSchemaReader and converter facades.""" - expected_exc = reader_exc if is_strict_reader else ValidationError - with pytest.raises(expected_exc, match="max_recursion_depth"): + with pytest.raises(ValidationError, match="max_recursion_depth"): constructor(max_recursion_depth=bad_depth) diff --git a/tests/cdm_data_loaders/converters/facades/test_pydantic_to_pyarrow_spec.py b/tests/cdm_data_loaders/converters/facades/test_pydantic_to_pyarrow_spec.py index 85db74dc..48f88abd 100644 --- a/tests/cdm_data_loaders/converters/facades/test_pydantic_to_pyarrow_spec.py +++ b/tests/cdm_data_loaders/converters/facades/test_pydantic_to_pyarrow_spec.py @@ -128,8 +128,15 @@ def test_generate_pyarrow_schema_pass_cases(case: Case, run: Run) -> None: assert empty_table.schema.equals(pa_schema, check_metadata=True) +@pytest.mark.parametrize( + "from_model", + [ + pytest.param(False, id="raw-dict"), + pytest.param(True, id="validated-model"), + ], +) @pytest.mark.parametrize(("case", "run"), RUN_PARAMS) -def test_coerce_pydantic_record_for_pyarrow_pass_cases(case: Case, run: Run) -> None: +def test_coerce_pydantic_record_for_pyarrow_pass_cases(case: Case, run: Run, from_model: bool) -> None: """Verify coerce_pydantic_record_for_pyarrow produces identical expected tables from dicts and validated BaseModels.""" model_cls = _load_model_class(case.schema) depth = _max_recursion_depth(run) @@ -137,20 +144,19 @@ def test_coerce_pydantic_record_for_pyarrow_pass_cases(case: Case, run: Run) -> pa_schema = generate_pyarrow_schema(model_cls, max_recursion_depth=depth, by_alias=by_alias) raw_records = load_json_records(run.data) - rows_from_dict = [ - coerce_pydantic_record_for_pyarrow(record, pa_schema, model=model_cls, by_alias=by_alias) - for record in raw_records - ] - assert_rows_and_table_match_expected( - rows_from_dict, pa_schema, run.expected_json, run.expected_table, table_attr=run.table_attr - ) - - rows_from_model = [ - coerce_pydantic_record_for_pyarrow(model_cls.model_validate(record), pa_schema, by_alias=by_alias) - for record in raw_records - ] + if from_model: + validated_cls = _load_pydantic_model_class(case.schema) + rows = [ + coerce_pydantic_record_for_pyarrow(validated_cls.model_validate(record), pa_schema, by_alias=by_alias) + for record in raw_records + ] + else: + rows = [ + coerce_pydantic_record_for_pyarrow(record, pa_schema, model=model_cls, by_alias=by_alias) + for record in raw_records + ] assert_rows_and_table_match_expected( - rows_from_model, pa_schema, run.expected_json, run.expected_table, table_attr=run.table_attr + rows, pa_schema, run.expected_json, run.expected_table, table_attr=run.table_attr ) diff --git a/tests/cdm_data_loaders/converters/sources/test_dlt.py b/tests/cdm_data_loaders/converters/sources/test_dlt.py index f017986f..28d38349 100644 --- a/tests/cdm_data_loaders/converters/sources/test_dlt.py +++ b/tests/cdm_data_loaders/converters/sources/test_dlt.py @@ -3,6 +3,7 @@ from typing import Any import pytest +from pydantic import ValidationError from cdm_data_loaders.converters.core.errors import ConversionError from cdm_data_loaders.converters.core.extensions import DEFAULT_EXTENSIONS, ExtensionError @@ -153,18 +154,16 @@ def test_read_fail_invalid_graph(source: dict[str, Any]) -> None: @pytest.mark.parametrize( - ("kwargs", "error_type", "match"), + ("kwargs", "match"), [ - ({"child_table_mode": "guess"}, ValueError, "child_table_mode"), - ({"include_dlt_columns": "yes"}, TypeError, None), + ({"child_table_mode": "guess"}, "child_table_mode"), + ({"include_dlt_columns": "yes"}, "include_dlt_columns"), ], ids=["invalid-child-table-mode", "invalid-include-dlt-columns"], ) -def test_dlt_reader_fail_invalid_options( - kwargs: dict[str, Any], error_type: type[Exception], match: str | None -) -> None: +def test_dlt_reader_fail_invalid_options(kwargs: dict[str, Any], match: str) -> None: """Invalid reader policies fail immediately.""" - with pytest.raises(error_type, match=match): + with pytest.raises(ValidationError, match=match): DltReader(**kwargs) diff --git a/tests/cdm_data_loaders/converters/sources/test_json_schema.py b/tests/cdm_data_loaders/converters/sources/test_json_schema.py index a29fe4ac..f308ea5e 100644 --- a/tests/cdm_data_loaders/converters/sources/test_json_schema.py +++ b/tests/cdm_data_loaders/converters/sources/test_json_schema.py @@ -58,26 +58,29 @@ def test_read_pass_exact_tree() -> None: inferred_type="object", properties=( Field( - "a", - TypedNode( + name="a", + node=TypedNode( type="unknown", nullable=True, declared_type=("string", "null"), source_keywords=("type",), - provenance=Provenance("json-schema", ("properties", "a")), + provenance=Provenance(source="json-schema", path=("properties", "a")), ), required=True, ), Field( - "b", TypedNode(type="never", nullable=False, provenance=Provenance("json-schema", ("properties", "b"))) + name="b", + node=TypedNode( + type="never", nullable=False, provenance=Provenance(source="json-schema", path=("properties", "b")) + ), ), ), required_names=("a", "not-declared"), additional_properties=TypedNode( - type="any", nullable=True, provenance=Provenance("json-schema", ("additionalProperties",)) + type="any", nullable=True, provenance=Provenance(source="json-schema", path=("additionalProperties",)) ), annotations={"$schema": DIALECT}, - provenance=Provenance("json-schema"), + provenance=Provenance(source="json-schema"), source_keywords=("$schema", "type", "required", "properties", "additionalProperties"), ) @@ -186,9 +189,9 @@ def test_read_fail_document(schema: dict[str, Any] | bool) -> None: def test_read_pass_custom_registry_and_empty_schemas() -> None: """Custom extensions and empty object subschemas need no nonempty required list.""" - registry = DEFAULT_EXTENSIONS.register(ExtensionSpec("x-owned", {"type": ["array", "null"]})) + registry = DEFAULT_EXTENSIONS.register(ExtensionSpec(namespace="x-owned", schema={"type": ["array", "null"]})) root = ( - JsonSchemaReader(registry) + JsonSchemaReader(extension_registry=registry) .read( { "$schema": DIALECT, diff --git a/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_ir.py b/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_ir.py index fad5ba0e..65f9cf38 100644 --- a/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_ir.py +++ b/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_ir.py @@ -7,7 +7,7 @@ import pyarrow as pa import pytest from linkml_runtime.linkml_model.meta import SchemaDefinition -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, ValidationError from pyspark.sql.types import StructType from cdm_data_loaders.converters.emitters.dlt import DltEmitter @@ -153,19 +153,17 @@ def test_pydantic_reader_pass_emit_to_pyarrow_pyspark_dlt_jsonschema_linkml( @pytest.mark.parametrize( - ("kwargs", "error_type", "match"), + ("kwargs", "match"), [ - ({"max_recursion_depth": 0}, ValueError, "max_recursion_depth"), - ({"max_recursion_depth": True}, TypeError, "max_recursion_depth"), - ({"by_alias": "yes"}, TypeError, "by_alias"), + ({"max_recursion_depth": 0}, "max_recursion_depth"), + ({"max_recursion_depth": True}, "max_recursion_depth"), + ({"by_alias": "yes"}, "by_alias"), ], ids=["non-positive-depth", "bool-depth", "non-bool-by-alias"], ) -def test_pydantic_reader_fail_invalid_configuration( - kwargs: dict[str, object], error_type: type[Exception], match: str -) -> None: - """PydanticReader validates max_recursion_depth and by_alias in __post_init__.""" - with pytest.raises(error_type, match=match): +def test_pydantic_reader_fail_invalid_configuration(kwargs: dict[str, object], match: str) -> None: + """PydanticReader validates max_recursion_depth and by_alias on construction.""" + with pytest.raises(ValidationError, match=match): PydanticReader(**kwargs) # type: ignore[arg-type] diff --git a/tests/cdm_data_loaders/converters/sources/test_xsd.py b/tests/cdm_data_loaders/converters/sources/test_xsd.py index 003dbef0..841dbbd6 100644 --- a/tests/cdm_data_loaders/converters/sources/test_xsd.py +++ b/tests/cdm_data_loaders/converters/sources/test_xsd.py @@ -8,6 +8,7 @@ import pyarrow as pa import pytest import xmlschema +from pydantic import ValidationError from pyspark.sql.types import StructType from cdm_data_loaders.converters.emitters.dlt import DltEmitter @@ -278,13 +279,13 @@ def test_xsd_to_pyarrow_facade_pass_outputs(tmp_path: Path, output_mode: str) -> [ pytest.param( functools.partial(XsdReader, max_recursion_depth=0), - ValueError, + ValidationError, "max_recursion_depth", id="zero-max-recursion-depth", ), pytest.param( functools.partial(XsdReader, max_recursion_depth=True), # type: ignore[arg-type] - TypeError, + ValidationError, "max_recursion_depth", id="bool-max-recursion-depth", ), diff --git a/tests/cdm_data_loaders/core/test_reporting.py b/tests/cdm_data_loaders/core/test_reporting.py index 8f8a8bad..1108997c 100644 --- a/tests/cdm_data_loaders/core/test_reporting.py +++ b/tests/cdm_data_loaders/core/test_reporting.py @@ -1,7 +1,6 @@ """Unit tests for `cdm_data_loaders.core.reporting`.""" import logging -from dataclasses import FrozenInstanceError from datetime import UTC, datetime from pathlib import Path from types import SimpleNamespace @@ -21,6 +20,7 @@ ParsedLoadJobFileName, TPackageJobState, ) +from pydantic import ValidationError from requests import RequestException from cdm_data_loaders.core.fields import FILE_LISTING_FILE_NAME, PIPELINE_DIR_NAME, REPORT_FILE_NAME @@ -442,10 +442,22 @@ def test_pipeline_report_frozen() -> None: started_at=MOCK_START_TIME, finished_at=MOCK_FINISH_TIME, ) - with pytest.raises(FrozenInstanceError): + with pytest.raises(ValidationError): report.pipeline_name = "other" # type: ignore[misc] +def test_pipeline_report_fail_invalid_load_info() -> None: + """PipelineReport rejects non-LoadInfo values for pipeline_load_info.""" + with pytest.raises(ValidationError, match="Input should be an instance of LoadInfo"): + PipelineReport( + pipeline_name=TEST_PIPELINE_NAME, + dataset_name="test_dataset", + started_at=MOCK_START_TIME, + finished_at=MOCK_FINISH_TIME, + pipeline_load_info={"loads_ids": ["1.0"]}, # type: ignore[arg-type] + ) + + def test_write_pipeline_report_pass_completed_no_errors(tmp_path: Path) -> None: """write_pipeline_report writes completed status and null pipeline_load_info_errs when there are no errors.""" report_path = tmp_path / "report_1.0.txt" @@ -454,7 +466,7 @@ def test_write_pipeline_report_pass_completed_no_errors(tmp_path: Path) -> None: dataset_name="test_dataset", started_at=MOCK_START_TIME, finished_at=MOCK_FINISH_TIME, - pipeline_load_info=MagicMock(loads_ids=["1700000000.123"]), + pipeline_load_info=MagicMock(spec=LoadInfo, loads_ids=["1700000000.123"], load_packages=[]), ) write_pipeline_report(report_path, report) @@ -483,6 +495,7 @@ def test_write_pipeline_report_pass_completed_with_errors_and_unsaved_info(tmp_p started_at=MOCK_START_TIME, finished_at=MOCK_FINISH_TIME, pipeline_load_info=MagicMock( + spec=LoadInfo, loads_ids=["1.0", "2.0"], load_packages=[ MagicMock( @@ -566,7 +579,7 @@ def test_write_pipeline_report_fail_oserror_logged( dataset_name="test_dataset", started_at=MOCK_START_TIME, finished_at=MOCK_FINISH_TIME, - pipeline_load_info=MagicMock(loads_ids=["1700000000.123"]), + pipeline_load_info=MagicMock(spec=LoadInfo, loads_ids=["1700000000.123"], load_packages=[]), ) monkeypatch.setattr(Path, "open", MagicMock(side_effect=OSError("disk full"))) @@ -595,7 +608,7 @@ def test_write_pipeline_report_pass_ingested_files( dataset_name="test_dataset", started_at=MOCK_START_TIME, finished_at=MOCK_FINISH_TIME, - pipeline_load_info=MagicMock(loads_ids=["1700000000.123"]), + pipeline_load_info=MagicMock(spec=LoadInfo, loads_ids=["1700000000.123"], load_packages=[]), ingested_files=ingested_files, ) @@ -640,7 +653,7 @@ def make_metadata_pipeline_and_load_info( else: pipeline.run.return_value = save_run_result - load_info = MagicMock() + load_info = MagicMock(spec=LoadInfo) load_info.has_failed_jobs = has_failed_jobs load_info.loads_ids = ["1700000000.123"] if loads_ids is None else loads_ids load_info.started_at = MOCK_LOAD_START_TIME @@ -984,7 +997,7 @@ def test_write_pipeline_report_pass_preprocess_errors( dataset_name=TEST_DATASET_NAME, started_at=MOCK_START_TIME, finished_at=MOCK_FINISH_TIME, - pipeline_load_info=MagicMock(loads_ids=["1700000000.123"], load_packages=load_packages), + pipeline_load_info=MagicMock(spec=LoadInfo, loads_ids=["1700000000.123"], load_packages=load_packages), preprocess_errors=preprocess_errors, ) @@ -1085,7 +1098,7 @@ def test_write_pipeline_report_pass_datetime_subclass_without_global_safe_dumper dataset_name=TEST_DATASET_NAME, started_at=started_at, finished_at=finished_at, - pipeline_load_info=MagicMock(loads_ids=["1700000000.123"]), + pipeline_load_info=MagicMock(spec=LoadInfo, loads_ids=["1700000000.123"], load_packages=[]), ) write_pipeline_report(report_path, report) diff --git a/tests/cdm_data_loaders/core/test_transforms.py b/tests/cdm_data_loaders/core/test_transforms.py index 86a4c155..2ee32a61 100644 --- a/tests/cdm_data_loaders/core/test_transforms.py +++ b/tests/cdm_data_loaders/core/test_transforms.py @@ -500,6 +500,13 @@ def test_validate_table_transforms_fail_unknown_tables( validate_table_transforms(settings, valid_tables) +@pytest.mark.parametrize( + "lookup_table", + [ + pytest.param("items", id="configured-table"), + pytest.param("other", id="unconfigured-table"), + ], +) @pytest.mark.parametrize( ( "use_transforms_module", @@ -544,6 +551,7 @@ def test_transform_settings_mixin_pass_loads_transforms( expected_transforms: TransformRegistry | None, expected_schema_map: ArrowSchemaRegistry | None, expected_item_transform: RecordTransformFn | None, + lookup_table: str, ) -> None: """TransformSettingsMixin loads TABLE_TRANSFORMS and OUTPUT_SCHEMAS when configured.""" mod_name = register_temp_module( @@ -567,10 +575,9 @@ def test_transform_settings_mixin_pass_loads_transforms( assert settings.output_schemas_module == output_schemas_module assert settings.table_transforms == expected_transforms assert settings.schema_map == expected_schema_map - assert settings.get_table_transforms("items") == ((expected_item_transform,) if expected_item_transform else ()) - assert settings.get_table_transforms("other") == () - assert settings.get_table_transform("items") is expected_item_transform - assert settings.get_table_transform("other") is None + expected_for_lookup = expected_item_transform if lookup_table == "items" else None + assert settings.get_table_transforms(lookup_table) == ((expected_for_lookup,) if expected_for_lookup else ()) + assert settings.get_table_transform(lookup_table) is expected_for_lookup @pytest.mark.parametrize( @@ -857,8 +864,17 @@ def test_emit_raw_table_fail_empty_prefix() -> None: emit_raw_table({"id": "1"}, ctx, prefix="") -def test_align_and_cast_table_casts_reorders_fills_missing_and_attaches_metadata() -> None: - """align_and_cast_table appends missing nullable columns, casts types, orders fields, and sets schema metadata.""" +@pytest.mark.parametrize( + "already_aligned", + [ + pytest.param(False, id="misaligned-casts-reorders-and-fills"), + pytest.param(True, id="already-aligned-fast-path"), + ], +) +def test_align_and_cast_table_casts_reorders_fills_missing_and_attaches_metadata( + already_aligned: bool, +) -> None: + """align_and_cast_table aligns mismatched tables and returns already-aligned tables unchanged.""" target_schema = pa.schema( [ pa.field("id", pa.string(), nullable=False, metadata={"comment": "Identifier"}), @@ -867,12 +883,16 @@ def test_align_and_cast_table_casts_reorders_fills_missing_and_attaches_metadata ], metadata={"comment": "Target table description"}, ) - raw_table = pa.table({"extra": ["drop"], "count": pa.array([3], type=pa.int32()), "id": ["a1"]}) - aligned = align_and_cast_table(raw_table, target_schema) + input_table = ( + pa.Table.from_pylist([{"id": "a1", "count": 3, "note": None}], schema=target_schema) + if already_aligned + else pa.table({"extra": ["drop"], "count": pa.array([3], type=pa.int32()), "id": ["a1"]}) + ) + aligned = align_and_cast_table(input_table, target_schema) assert aligned.schema.equals(target_schema, check_metadata=True) assert aligned.to_pylist() == [{"id": "a1", "count": 3, "note": None}] - # fast-path returns the exact same table instance when schema and metadata already match - assert align_and_cast_table(aligned, target_schema) is aligned + if already_aligned: + assert aligned is input_table def test_compose_transforms_fails_with_no_transforms() -> None: diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py index eae6041e..babffba2 100644 --- a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py +++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py @@ -681,21 +681,35 @@ def test_filter_unpromoted_manifest_lines_exact_version_and_unparseable() -> Non ] +@pytest.mark.parametrize( + ("database", "expected_ids"), + [ + pytest.param(Source.REFSEQ, ["GCF_000001215.4"], id="refseq"), + pytest.param(Source.GENBANK, ["GCA_012345678.1"], id="genbank"), + ], +) @patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.list_objects") -def test_scan_store_filters_by_database_and_rejects_unknown_database(mock_list_objects: MagicMock) -> None: - """Scan store filters by GCA_ prefix for genbank and raises ValueError on unknown database.""" +def test_scan_store_filters_by_database( + mock_list_objects: MagicMock, + database: Source, + expected_ids: list[str], +) -> None: + """Filter S3 object keys by database accession prefix when scanning synthetic summary.""" mock_list_objects.return_value = [ {"Key": "prefix/GCF_000001215.4_Release_6/file.gz", "LastModified": datetime(2024, 1, 1, tzinfo=UTC)}, {"Key": "prefix/GCA_012345678.1_ASM123v1/file.gz", "LastModified": datetime(2024, 1, 2, tzinfo=UTC)}, ] - genbank_records = scan_store_to_synthetic_summary( + records = scan_store_to_synthetic_summary( PurePosixPath("test-bucket"), PurePosixPath("prefix"), "2024/01/31", - database=Source.GENBANK, + database=database, ) - assert list(genbank_records) == ["GCA_012345678.1"] + assert list(records) == expected_ids + +def test_scan_store_rejects_unknown_database() -> None: + """Raise ValueError when scanning store with an unsupported database name.""" with pytest.raises(ValueError, match="Unknown database"): scan_store_to_synthetic_summary( PurePosixPath("test-bucket"), diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_updates.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_updates.py index d61522b3..26c0cab1 100644 --- a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_updates.py +++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_updates.py @@ -245,14 +245,31 @@ def test_compute_batch_width(row_counts: tuple[int, int], batch_size: int, expec assert compute_batch_width(splits, batch_size) == expected_width -def test_write_id_batches_writes_zero_padded_files(tmp_path: Path) -> None: +@pytest.mark.parametrize( + ("ids", "prefix", "expected_written", "expected_files"), + [ + pytest.param([], "empty_prefix", 0, {}, id="empty-ids"), + pytest.param( + ["GCF_000000001.1", "GCF_000000002.1", "GCF_000000003.1"], + "after_20240501", + 2, + { + "after_20240501-01.txt": "GCF_000000001.1\nGCF_000000002.1\n", + "after_20240501-02.txt": "GCF_000000003.1\n", + }, + id="zero-padded-batches", + ), + ], +) +def test_write_id_batches_writes_zero_padded_files( + tmp_path: Path, + ids: list[str], + prefix: str, + expected_written: int, + expected_files: dict[str, str], +) -> None: """Write numbered batch files with zero-padded suffixes and skip writing when ids is empty.""" - assert write_id_batches([], tmp_path, "empty_prefix", batch_size=2, width=2) == 0 - assert list(tmp_path.iterdir()) == [] - - ids = ["GCF_000000001.1", "GCF_000000002.1", "GCF_000000003.1"] - written = write_id_batches(ids, tmp_path, "after_20240501", batch_size=2, width=2) - expected_batches = 2 - assert written == expected_batches - assert (tmp_path / "after_20240501-01.txt").read_text(encoding="utf-8") == "GCF_000000001.1\nGCF_000000002.1\n" - assert (tmp_path / "after_20240501-02.txt").read_text(encoding="utf-8") == "GCF_000000003.1\n" + written = write_id_batches(ids, tmp_path, prefix, batch_size=2, width=2) + assert written == expected_written + actual_files = {p.name: p.read_text(encoding="utf-8") for p in sorted(tmp_path.iterdir())} + assert actual_files == expected_files diff --git a/tests/cdm_data_loaders/datasets/uniprot/test_common.py b/tests/cdm_data_loaders/datasets/uniprot/test_common.py index 25b990ce..9813b698 100644 --- a/tests/cdm_data_loaders/datasets/uniprot/test_common.py +++ b/tests/cdm_data_loaders/datasets/uniprot/test_common.py @@ -3,8 +3,8 @@ import datetime as dt import json import logging -from collections.abc import Mapping -from typing import Any +from collections.abc import Callable, Mapping +from typing import Any, Final import pytest @@ -110,13 +110,23 @@ def test_format_curie_translates_prefixes_and_applies_fallback( assert format_curie(db, accession, fallback_upper=fallback_upper) == expected -def test_resolve_timestamp_returns_explicit_or_utc_now() -> None: - """resolve_timestamp preserves an explicit timestamp and generates a UTC timestamp when None.""" - fixed = dt.datetime(2025, 3, 1, 8, 0, tzinfo=dt.UTC) - assert resolve_timestamp(fixed) == fixed +FIXED_UTC_TS: Final[dt.datetime] = dt.datetime(2025, 3, 1, 8, 0, tzinfo=dt.UTC) + - generated = resolve_timestamp(None) - assert generated.tzinfo == dt.UTC +@pytest.mark.parametrize( + "explicit_ts", + [ + pytest.param(FIXED_UTC_TS, id="explicit-timestamp"), + pytest.param(None, id="none-defaults-to-utc-now"), + ], +) +def test_resolve_timestamp_returns_explicit_or_utc_now(explicit_ts: dt.datetime | None) -> None: + """resolve_timestamp preserves an explicit timestamp and generates a UTC timestamp when None.""" + resolved = resolve_timestamp(explicit_ts) + if explicit_ts is not None: + assert resolved == explicit_ts + else: + assert resolved.tzinfo == dt.UTC @pytest.mark.parametrize( @@ -154,42 +164,84 @@ def test_reject_cdm_entry_preserves_context_provenance( } +def _parse_ok_entry() -> dict[str, list[dict[str, Any]]]: + """Return a parsed single-entity row mapping.""" + return {"entity": [{"entity_id": "uniprot:P1"}]} + + +def _parse_fail_entry() -> dict[str, list[dict[str, Any]]]: + """Raise a RuntimeError to exercise exception routing.""" + msg = "explode" + raise RuntimeError(msg) + + +@pytest.mark.parametrize( + ("entry", "parse_fn", "expected", "expected_log_substr"), + [ + pytest.param( + {"_id": "ok"}, + _parse_ok_entry, + {"entity": [{"entity_id": "uniprot:P1"}]}, + None, + id="parsed-ok", + ), + pytest.param( + {"_id": "broken"}, + _parse_fail_entry, + { + REJECTED_TABLE_NAME: [ + { + "table_name": "entry", + "source_file": "batch/in.xml.gz", + "line_no": None, + "raw_record": json.dumps({"_id": "broken"}), + "error_detail": "explode", + } + ] + }, + "Unexpected parse failure", + id="exception-routed-to-rejected", + ), + ], +) def test_run_cdm_entry_parser_returns_parsed_or_routes_exception_with_ctx( caplog: pytest.LogCaptureFixture, + entry: dict[str, Any], + parse_fn: Callable[[], dict[str, list[dict[str, Any]]]], + expected: dict[str, list[dict[str, Any]]], + expected_log_substr: str | None, ) -> None: """run_cdm_entry_parser returns parse_fn output on success and routes exceptions to _rejected with ctx.""" test_logger = logging.getLogger("test_uniprot_common") ctx = TransformContext(table_name="entry", source_file="batch/in.xml.gz") - ok = run_cdm_entry_parser( - {"_id": "ok"}, - lambda: {"entity": [{"entity_id": "uniprot:P1"}]}, - ctx=ctx, - logger=test_logger, - error_message="Unexpected parse failure", - ) - assert ok == {"entity": [{"entity_id": "uniprot:P1"}]} - - def _fail() -> dict[str, list[dict[str, Any]]]: - msg = "explode" - raise RuntimeError(msg) - with caplog.at_level(logging.ERROR, logger="test_uniprot_common"): - failed = run_cdm_entry_parser( - {"_id": "broken"}, - _fail, + result = run_cdm_entry_parser( + entry, + parse_fn, ctx=ctx, logger=test_logger, error_message="Unexpected parse failure", ) - assert "Unexpected parse failure" in caplog.text - assert failed[REJECTED_TABLE_NAME][0]["source_file"] == "batch/in.xml.gz" - assert failed[REJECTED_TABLE_NAME][0]["table_name"] == "entry" - assert failed[REJECTED_TABLE_NAME][0]["error_detail"] == "explode" + assert result == expected + if expected_log_substr is not None: + assert expected_log_substr in caplog.text + else: + assert caplog.records == [] -def test_make_cdm_entry_transform_supports_single_arg_and_context_invocation() -> None: +@pytest.mark.parametrize( + ("extra_args", "expected_source"), + [ + pytest.param((), None, id="without-ctx"), + pytest.param((TransformContext(table_name="entry", source_file="chunk.xml"),), "chunk.xml", id="with-ctx"), + ], +) +def test_make_cdm_entry_transform_supports_single_arg_and_context_invocation( + extra_args: tuple[TransformContext, ...], + expected_source: str | None, +) -> None: """make_cdm_entry_transform produces a context-aware RecordTransformFn callable with or without ctx.""" def _entry_fn(entry: Mapping[str, Any], ctx: TransformContext | None) -> dict[str, list[dict[str, Any]]]: @@ -206,7 +258,4 @@ def _entry_fn(entry: Mapping[str, Any], ctx: TransformContext | None) -> dict[st _, _, takes_ctx = inspect_transform_signature(transform_fn) assert takes_ctx is True - assert transform_fn({"id": "P1"}) == {"entity": [{"id": "P1", "source": None}]} - assert transform_fn({"id": "P1"}, TransformContext(table_name="entry", source_file="chunk.xml")) == { - "entity": [{"id": "P1", "source": "chunk.xml"}] - } + assert transform_fn({"id": "P1"}, *extra_args) == {"entity": [{"id": "P1", "source": expected_source}]} diff --git a/tests/cdm_data_loaders/datasets/uniprot/test_kb.py b/tests/cdm_data_loaders/datasets/uniprot/test_uniprot_kb.py similarity index 98% rename from tests/cdm_data_loaders/datasets/uniprot/test_kb.py rename to tests/cdm_data_loaders/datasets/uniprot/test_uniprot_kb.py index 2d67922e..5631b782 100644 --- a/tests/cdm_data_loaders/datasets/uniprot/test_kb.py +++ b/tests/cdm_data_loaders/datasets/uniprot/test_uniprot_kb.py @@ -8,12 +8,12 @@ import pytest from lxml.etree import XMLParser, fromstring -import cdm_data_loaders.datasets.uniprot.kb as uniprot_kb_module +import cdm_data_loaders.datasets.uniprot.uniprot_kb as uniprot_kb_module from cdm_data_loaders.coercion.xml import make_xml_element_converter from cdm_data_loaders.core.fields import REJECTED_TABLE_NAME from cdm_data_loaders.core.transforms.introspection import inspect_transform_signature from cdm_data_loaders.core.transforms.routing import route_and_coerce_record -from cdm_data_loaders.datasets.uniprot.kb import ( +from cdm_data_loaders.datasets.uniprot.uniprot_kb import ( DB, DESCRIPTION, ENTITY_ID, @@ -1074,10 +1074,8 @@ def test_make_uniprot_cdm_transform() -> None: ] -def test_make_uniprot_cdm_transform_propagates_source_file_to_rejected_and_logs_error( - caplog: pytest.LogCaptureFixture, -) -> None: - """make_uniprot_cdm_transform resolves a default UTC timestamp and preserves source_file on _rejected.""" +def test_make_uniprot_cdm_transform_defaults_updated_to_utc() -> None: + """make_uniprot_cdm_transform exposes a context-aware signature and defaults updated to UTC.""" transform_fn = make_uniprot_cdm_transform() assert inspect_transform_signature(transform_fn) == (False, True, True) @@ -1086,6 +1084,13 @@ def test_make_uniprot_cdm_transform_propagates_source_file_to_rejected_and_logs_ assert isinstance(updated, dt.datetime) assert updated.tzinfo == dt.UTC + +def test_make_uniprot_cdm_transform_propagates_source_file_to_rejected_and_logs_error( + caplog: pytest.LogCaptureFixture, +) -> None: + """make_uniprot_cdm_transform preserves source_file on _rejected rows and logs an ERROR.""" + transform_fn = make_uniprot_cdm_transform() + with caplog.at_level(logging.ERROR, logger="cdm_data_loaders.datasets.uniprot.kb"): routed = route_and_coerce_record( {"_dataset": "Swiss-Prot", "accession": []}, diff --git a/tests/cdm_data_loaders/datasets/uniprot/test_uniref.py b/tests/cdm_data_loaders/datasets/uniprot/test_uniref.py index 372c4aac..f88c6f0e 100644 --- a/tests/cdm_data_loaders/datasets/uniprot/test_uniref.py +++ b/tests/cdm_data_loaders/datasets/uniprot/test_uniref.py @@ -755,10 +755,8 @@ def test_make_uniref_cdm_transform() -> None: ] -def test_make_uniref_cdm_transform_propagates_source_file_to_rejected( - caplog: pytest.LogCaptureFixture, -) -> None: - """make_uniref_cdm_transform resolves a default UTC timestamp and preserves source_file on _rejected.""" +def test_make_uniref_cdm_transform_defaults_updated_to_utc() -> None: + """make_uniref_cdm_transform exposes a context-aware signature and defaults updated to UTC.""" transform_fn = make_uniref_cdm_transform(PROTOCOL) assert inspect_transform_signature(transform_fn) == (False, True, True) @@ -767,6 +765,13 @@ def test_make_uniref_cdm_transform_propagates_source_file_to_rejected( assert isinstance(updated, dt.datetime) assert updated.tzinfo == dt.UTC + +def test_make_uniref_cdm_transform_propagates_source_file_to_rejected( + caplog: pytest.LogCaptureFixture, +) -> None: + """make_uniref_cdm_transform preserves source_file on _rejected rows and logs an ERROR.""" + transform_fn = make_uniref_cdm_transform(PROTOCOL) + with caplog.at_level(logging.ERROR, logger="cdm_data_loaders.datasets.uniprot.uniref"): routed = route_and_coerce_record( {"name": "Cluster: missing id"}, diff --git a/tests/cdm_data_loaders/io/s3/test_object_utils.py b/tests/cdm_data_loaders/io/s3/test_object_utils.py index 5a88acd0..aecc2fc8 100644 --- a/tests/cdm_data_loaders/io/s3/test_object_utils.py +++ b/tests/cdm_data_loaders/io/s3/test_object_utils.py @@ -1372,16 +1372,25 @@ def test_is_not_found_client_error(code: str, status: int, expected: bool) -> No @pytest.mark.s3 +@pytest.mark.parametrize( + ("key", "seed_body", "expected"), + [ + pytest.param("present/data.bin", b"present-bytes", b"present-bytes", id="existing-object"), + pytest.param("missing/data.bin", None, None, id="missing-object"), + ], +) def test_get_object_bytes_if_exists_existing_and_missing( mock_s3_client: S3Client, caplog: pytest.LogCaptureFixture, + key: str, + seed_body: bytes | None, + expected: bytes | None, ) -> None: """Return payload bytes when present and None without ERROR logs when absent.""" - mock_s3_client.put_object(Bucket=TEST_BUCKET, Key="present/data.bin", Body=b"present-bytes") - assert get_object_bytes_if_exists(f"{TEST_BUCKET}/present/data.bin") == b"present-bytes" - + if seed_body is not None: + mock_s3_client.put_object(Bucket=TEST_BUCKET, Key=key, Body=seed_body) with caplog.at_level(logging.ERROR): - assert get_object_bytes_if_exists(f"{TEST_BUCKET}/missing/data.bin") is None + assert get_object_bytes_if_exists(f"{TEST_BUCKET}/{key}") == expected assert caplog.records == [] diff --git a/tests/cdm_data_loaders/io/test_ftp_client.py b/tests/cdm_data_loaders/io/test_ftp_client.py index a515c956..41749937 100644 --- a/tests/cdm_data_loaders/io/test_ftp_client.py +++ b/tests/cdm_data_loaders/io/test_ftp_client.py @@ -217,20 +217,32 @@ def test_close_all(mock_connect: MagicMock) -> None: mock_ftp.quit.assert_called_once() +@pytest.mark.parametrize( + "borrow_existing", + [ + pytest.param(False, id="owned-closes-on-exit"), + pytest.param(True, id="borrowed-left-open"), + ], +) @patch("cdm_data_loaders.io.ftp_client.connect_ftp") -def test_ftp_session_owned_vs_borrowed_lifecycle(mock_connect: MagicMock) -> None: +def test_ftp_session_owned_vs_borrowed_lifecycle( + mock_connect: MagicMock, + borrow_existing: bool, +) -> None: """Close FTP connection on exit when owned, and leave caller-supplied FTP open.""" - owned_ftp = MagicMock(spec=FTP) - mock_connect.return_value = owned_ftp - with ftp_session("ftp.example.com", timeout=FTP_TIMEOUT) as active: - assert active is owned_ftp - mock_connect.assert_called_once_with("ftp.example.com", timeout=FTP_TIMEOUT) - owned_ftp.quit.assert_called_once() - - borrowed_ftp = MagicMock(spec=FTP) - with ftp_session("ftp.example.com", ftp=borrowed_ftp) as active: - assert active is borrowed_ftp - borrowed_ftp.quit.assert_not_called() + expected_ftp = MagicMock(spec=FTP) + mock_connect.return_value = expected_ftp + ftp_arg = expected_ftp if borrow_existing else None + + with ftp_session("ftp.example.com", timeout=FTP_TIMEOUT, ftp=ftp_arg) as active: + assert active is expected_ftp + + if borrow_existing: + mock_connect.assert_not_called() + expected_ftp.quit.assert_not_called() + else: + mock_connect.assert_called_once_with("ftp.example.com", timeout=FTP_TIMEOUT) + expected_ftp.quit.assert_called_once() @pytest.mark.parametrize( diff --git a/tests/cdm_data_loaders/pipelines/conftest.py b/tests/cdm_data_loaders/pipelines/conftest.py index 651f0286..bea349a2 100644 --- a/tests/cdm_data_loaders/pipelines/conftest.py +++ b/tests/cdm_data_loaders/pipelines/conftest.py @@ -1,9 +1,11 @@ """Shared unit-test fixtures for ``tests/cdm_data_loaders/pipelines``.""" +from datetime import UTC, datetime from unittest.mock import MagicMock import dlt import pytest +from dlt.common.pipeline import LoadInfo from cdm_data_loaders.pipelines import runner @@ -32,8 +34,19 @@ def mock_dlt(monkeypatch: pytest.MonkeyPatch) -> MagicMock: :rtype: MagicMock """ mock = MagicMock() - mock.pipeline.return_value.runtime_config.slack_incoming_hook = None - mock.pipeline.return_value.run.return_value = MagicMock() + mock_pipeline = mock.pipeline.return_value + mock_pipeline.pipeline_name = "mock_pipeline" + mock_pipeline.dataset_name = "mock_dataset" + mock_pipeline.runtime_config.slack_incoming_hook = None + mock_pipeline.last_trace.started_at = datetime(2025, 1, 1, 0, 0, 0, tzinfo=UTC) + mock_pipeline.last_trace.finished_at = datetime(2025, 1, 1, 0, 0, 1, tzinfo=UTC) + mock_load_info = MagicMock(spec=LoadInfo) + mock_load_info.__len__.return_value = 1 + mock_load_info.has_failed_jobs = False + mock_load_info.loads_ids = ["1.0"] + mock_load_info.load_packages = [] + mock_load_info.asdict.return_value = {"loads_ids": ["1.0"]} + mock_pipeline.run.return_value = mock_load_info monkeypatch.setattr(dlt, "destination", mock.destination) monkeypatch.setattr(dlt, "pipeline", mock.pipeline) return mock diff --git a/tests/cdm_data_loaders/pipelines/presets/test_uniprot_cdm.py b/tests/cdm_data_loaders/pipelines/presets/test_uniprot_cdm.py index 80ef1406..cbbed28e 100644 --- a/tests/cdm_data_loaders/pipelines/presets/test_uniprot_cdm.py +++ b/tests/cdm_data_loaders/pipelines/presets/test_uniprot_cdm.py @@ -10,7 +10,7 @@ from cdm_data_loaders.core.cli import CliRunner from cdm_data_loaders.core.registry import TransformRegistry -from cdm_data_loaders.datasets.uniprot.kb import UNIPROT_CDM_SCHEMA_MAP +from cdm_data_loaders.datasets.uniprot.uniprot_kb import UNIPROT_CDM_SCHEMA_MAP from cdm_data_loaders.datasets.uniprot.uniref import UNIREF_CDM_SCHEMA_MAP from cdm_data_loaders.pipelines.presets.uniprot_cdm import ( PIPELINE_NAME, diff --git a/tests/cdm_data_loaders/pipelines/test_runner.py b/tests/cdm_data_loaders/pipelines/test_runner.py index d8a2f659..c93c6a2f 100644 --- a/tests/cdm_data_loaders/pipelines/test_runner.py +++ b/tests/cdm_data_loaders/pipelines/test_runner.py @@ -12,6 +12,7 @@ import pytest import yaml from dlt.common.destination.exceptions import DestinationTerminalException +from dlt.common.pipeline import LoadInfo from dlt.common.schema.typing import TTableSchema from dlt.common.typing import TDataItems from dlt.destinations.impl.filesystem.filesystem import FilesystemLoadJob @@ -94,7 +95,8 @@ def make_mock_pipeline(save_run_result: object, slack_hook: str | None = None) - pipeline.runtime_config.slack_incoming_hook = slack_hook pipeline.last_trace.started_at = datetime(2025, 1, 2, 3, 4, 0, tzinfo=UTC) pipeline.last_trace.finished_at = datetime(2025, 1, 2, 3, 4, 10, tzinfo=UTC) - entity_load_info = MagicMock() + entity_load_info = MagicMock(spec=LoadInfo) + entity_load_info.__len__.return_value = 1 entity_load_info.has_failed_jobs = False entity_load_info.loads_ids = ["1.0"] entity_load_info.started_at = datetime(2025, 1, 2, 3, 4, 5, tzinfo=UTC) diff --git a/tests/cdm_data_loaders/utils/test_checksums.py b/tests/cdm_data_loaders/utils/test_checksums.py index 78ff6c88..c4e1ac46 100644 --- a/tests/cdm_data_loaders/utils/test_checksums.py +++ b/tests/cdm_data_loaders/utils/test_checksums.py @@ -2,7 +2,6 @@ import base64 import copy -import dataclasses import hashlib from collections.abc import Callable, Generator from io import BytesIO @@ -13,6 +12,7 @@ import requests from _pytest.mark.structures import ParameterSet from awscrt.checksums import crc64nvme as _crc64nvme +from pydantic import ValidationError from cdm_data_loaders.utils.checksums import ( EXTRA_ALGORITHMS, @@ -84,7 +84,7 @@ def test_extra_algorithms_pass_contains_only_crc64nvme() -> None: assert frozenset({"crc64nvme"}) == EXTRA_ALGORITHMS -"""Tests for the ChecksumEntry dataclass""" +"""Tests for the ChecksumEntry model""" def test_checksumentry_pass_construction_and_fields() -> None: @@ -117,22 +117,19 @@ def test_checksumentry_pass_hashable() -> None: def test_checksumentry_fail_frozen_immutable() -> None: - """Mutating an existing field on a ChecksumEntry raises FrozenInstanceError.""" + """Mutating an existing field on a ChecksumEntry raises ValidationError.""" entry = ChecksumEntry(algorithm="sha256", value="flash_gordon") - with pytest.raises(dataclasses.FrozenInstanceError, match="cannot assign to field 'algorithm'"): + with pytest.raises(ValidationError, match="Instance is frozen"): entry.algorithm = "md5" # type: ignore[misc] def test_checksumentry_fail_frozen_prevents_new_attributes() -> None: - """The frozen dataclass rejects attribute assignment even for names that aren't declared fields.""" + """The frozen model rejects extra attributes both at construction and on assignment.""" entry = ChecksumEntry(algorithm="sha256", value="flash_gordon") - # can be one of two errors, depending on which version of CPython is in use. - # should be a FrozenInstanceError but some versions produce a TypeError instead - with pytest.raises( - (TypeError, dataclasses.FrozenInstanceError), - match=r"(is not an instance or subtype of type \(ChecksumEntry\)|cannot assign to field 'extra')", - ): + with pytest.raises(ValidationError): entry.extra = "not allowed" # type: ignore[attr-defined] + with pytest.raises(ValidationError, match="Extra inputs are not permitted"): + ChecksumEntry(algorithm="sha256", value="flash_gordon", extra="not allowed") # type: ignore[call-arg] """Tests of new_hasher""" @@ -595,14 +592,22 @@ def test_parse_md5_checksums_file_parses_gnu_manifest() -> None: } -def test_finalize_digest_formats_hex_and_crc64nvme_base64() -> None: +STREAMED_PAYLOAD: Final[bytes] = b"streamed-payload" + + +@pytest.mark.parametrize( + ("algorithm", "expected"), + [ + pytest.param("sha256", hashlib.sha256(STREAMED_PAYLOAD).hexdigest(), id="sha256-hex"), + pytest.param( + "crc64nvme", + base64.b64encode(_crc64nvme(STREAMED_PAYLOAD, 0).to_bytes(8, "big")).decode(), + id="crc64nvme-base64", + ), + ], +) +def test_finalize_digest_formats_hex_and_crc64nvme_base64(algorithm: str, expected: str) -> None: """Return hex digests for hashlib hashers and base64 digests for Crc64NvmeHasher.""" - payload = b"streamed-payload" - sha_hasher = new_hasher("sha256") - sha_hasher.update(payload) - assert finalize_digest(sha_hasher) == hashlib.sha256(payload).hexdigest() - - crc_hasher = new_hasher("crc64nvme") - crc_hasher.update(payload) - expected_crc = _crc64nvme(payload, 0) - assert finalize_digest(crc_hasher) == base64.b64encode(expected_crc.to_bytes(8, "big")).decode() + hasher = new_hasher(algorithm) + hasher.update(STREAMED_PAYLOAD) + assert finalize_digest(hasher) == expected diff --git a/tests/cdm_data_loaders/utils/test_progress.py b/tests/cdm_data_loaders/utils/test_progress.py index 74043631..f71c3eed 100644 --- a/tests/cdm_data_loaders/utils/test_progress.py +++ b/tests/cdm_data_loaders/utils/test_progress.py @@ -3,6 +3,7 @@ import re import threading import time +from collections.abc import Callable import pytest from tqdm import tqdm @@ -17,51 +18,68 @@ """Tests for SynchronizedCallback""" -def test_synchronizedcallback_pass_forwards_single_call() -> None: - """Calling the wrapper invokes the wrapped callback exactly once with the given argument.""" +def _make_builtin_method_sink() -> tuple[Callable[[int], None], list[int]]: + """Return a list.append callback and its recorded argument list.""" calls: list[int] = [] - wrapped = SynchronizedCallback(calls.append) - wrapped(42) - assert calls == [42] + return calls.append, calls -def test_synchronizedcallback_pass_forwards_multiple_calls_in_order() -> None: - """Sequential calls are forwarded to the wrapped callback in the order they were made.""" +def _make_named_function_sink() -> tuple[Callable[[int], None], list[int]]: + """Return a named function callback and its recorded argument list.""" calls: list[int] = [] - wrapped = SynchronizedCallback(calls.append) - for amount in (1, 2, 3, 4): - wrapped(amount) - assert calls == [1, 2, 3, 4] + def record(amount: int) -> None: + calls.append(amount) + + return record, calls -def test_synchronizedcallback_pass_works_with_lambda() -> None: - """SynchronizedCallback works with a lambda, not just named functions.""" - n1 = 5 - n2 = 7 - total = {"value": 0} - wrapped = SynchronizedCallback(lambda n: total.__setitem__("value", total["value"] + n)) - wrapped(n1) - wrapped(n2) - assert total["value"] == n1 + n2 + +def _make_lambda_sink() -> tuple[Callable[[int], None], list[int]]: + """Return a lambda callback and its recorded argument list.""" + calls: list[int] = [] + return (lambda amount: calls.extend([amount])), calls -def test_synchronizedcallback_pass_works_with_bound_method() -> None: - """SynchronizedCallback works with a bound method as the wrapped callable.""" - n1 = 5 - n2 = 7 +def _make_bound_method_sink() -> tuple[Callable[[int], None], list[int]]: + """Return a bound instance method callback and its recorded argument list.""" class Accumulator: def __init__(self) -> None: - self.total = 0 + self.calls: list[int] = [] def add(self, amount: int) -> None: - self.total += amount + self.calls.append(amount) acc = Accumulator() - wrapped = SynchronizedCallback(acc.add) - wrapped(n1) - wrapped(n2) - assert acc.total == n1 + n2 + return acc.add, acc.calls + + +@pytest.mark.parametrize( + "sink_factory", + [ + pytest.param(_make_builtin_method_sink, id="builtin-method"), + pytest.param(_make_named_function_sink, id="named-function"), + pytest.param(_make_lambda_sink, id="lambda"), + pytest.param(_make_bound_method_sink, id="bound-method"), + ], +) +def test_synchronizedcallback_pass_forwards_single_call( + sink_factory: Callable[[], tuple[Callable[[int], None], list[int]]], +) -> None: + """Calling the wrapper invokes the wrapped callback once across callable kinds.""" + callback, calls = sink_factory() + wrapped = SynchronizedCallback(callback) + wrapped(42) + assert calls == [42] + + +def test_synchronizedcallback_pass_forwards_multiple_calls_in_order() -> None: + """Sequential calls are forwarded to the wrapped callback in the order they were made.""" + calls: list[int] = [] + wrapped = SynchronizedCallback(calls.append) + for amount in (1, 2, 3, 4): + wrapped(amount) + assert calls == [1, 2, 3, 4] def test_synchronizedcallback_pass_thread_safe_concurrent_calls_all_counted() -> None: diff --git a/tests/cdm_data_loaders/utils/test_recursion.py b/tests/cdm_data_loaders/utils/test_recursion.py index b4710aef..7e830d81 100644 --- a/tests/cdm_data_loaders/utils/test_recursion.py +++ b/tests/cdm_data_loaders/utils/test_recursion.py @@ -38,18 +38,18 @@ def test_validate_max_recursion_depth_pass_valid_depths(depth: int) -> None: assert validate_max_recursion_depth(depth) == depth -@pytest.mark.parametrize( - ("bad_value", "name", "exc_cls", "match"), - [ - pytest.param(0, "max_recursion_depth", ValueError, "max_recursion_depth must be >= 1", id="zero"), - pytest.param(-3, "max_ref_depth", ValueError, "max_ref_depth must be >= 1", id="negative"), - pytest.param(True, "max_recursion_depth", TypeError, "must be an integer >= 1", id="bool-true"), - pytest.param(False, "max_ref_depth", TypeError, "must be an integer >= 1", id="bool-false"), - pytest.param(1.5, "max_recursion_depth", TypeError, "must be an integer >= 1", id="float"), - pytest.param("2", "max_recursion_depth", TypeError, "must be an integer >= 1", id="string"), - pytest.param(None, "max_recursion_depth", TypeError, "must be an integer >= 1", id="none"), - ], -) +positive_int_params = [ + pytest.param(0, "max_recursion_depth", ValueError, "must be >= 1", id="zero"), + pytest.param(-3, "max_ref_depth", ValueError, "must be >= 1", id="negative"), + pytest.param(True, "max_recursion_depth", TypeError, "must be an integer >= 1", id="bool-true"), + pytest.param(False, "max_ref_depth", TypeError, "must be an integer >= 1", id="bool-false"), + pytest.param(1.5, "max_recursion_depth", TypeError, "must be an integer >= 1", id="float"), + pytest.param("2", "max_recursion_depth", TypeError, "must be an integer >= 1", id="string"), + pytest.param(None, "max_recursion_depth", TypeError, "must be an integer >= 1", id="none"), +] + + +@pytest.mark.parametrize(("bad_value", "name", "exc_cls", "match"), positive_int_params) def test_validate_positive_int_param_fail_invalid_values( bad_value: object, name: str, @@ -57,7 +57,7 @@ def test_validate_positive_int_param_fail_invalid_values( match: str, ) -> None: """Reject booleans, non-integers, zero, and negative integers.""" - with pytest.raises(exc_cls, match=match): + with pytest.raises(exc_cls, match=f"{name} {match}"): validate_positive_int_param(bad_value, name) # type: ignore[arg-type] @@ -77,17 +77,21 @@ def test_strict_positive_recursion_depth_pydantic_field_pass_valid_inputs( @pytest.mark.parametrize( - "bad_depth", + ("bad_value", "match"), [ - pytest.param(0, id="zero"), - pytest.param(-1, id="negative"), - pytest.param(True, id="bool-true"), - pytest.param(False, id="bool-false"), - pytest.param("2", id="numeric-string"), - pytest.param(2.0, id="float"), + pytest.param(0, "Input should be greater than 0", id="zero"), + pytest.param(-3, "Input should be greater than 0", id="negative"), + pytest.param(True, "Input should be a valid integer", id="bool-true"), + pytest.param(False, "Input should be a valid integer", id="bool-false"), + pytest.param(1.5, "Input should be a valid integer", id="float"), + pytest.param("2", "Input should be a valid integer", id="string"), + pytest.param(None, "Input should be a valid integer", id="none"), ], ) -def test_strict_positive_recursion_depth_pydantic_field_fail_invalid_inputs(bad_depth: object) -> None: +def test_strict_positive_recursion_depth_pydantic_field_fail_invalid_inputs( + bad_value: object, + match: str, +) -> None: """Reject booleans, strings, floats, zero, and negative values in Pydantic models.""" - with pytest.raises(ValidationError): - _DepthModel(depth=bad_depth) # type: ignore[arg-type] + with pytest.raises(ValidationError, match=match): + _DepthModel(depth=bad_value) # type: ignore[arg-type] diff --git a/tests/cdm_data_loaders/validation/jsonschema/test_models.py b/tests/cdm_data_loaders/validation/jsonschema/test_models.py index 11ac9970..b2777f70 100644 --- a/tests/cdm_data_loaders/validation/jsonschema/test_models.py +++ b/tests/cdm_data_loaders/validation/jsonschema/test_models.py @@ -3,6 +3,7 @@ import json import re from collections.abc import Callable +from functools import partial from pathlib import Path from typing import Any, Final @@ -330,6 +331,13 @@ def test_custom_xsv_metaschema_pass_accepts_valid_config( assert result.jsonschema == schema +@pytest.mark.parametrize( + "via_file", + [ + pytest.param(False, id="validator-check-schema"), + pytest.param(True, id="validate-xsv-jsonschema"), + ], +) @pytest.mark.parametrize( ("x_xsv_config", "error_match"), [ @@ -350,20 +358,25 @@ def test_custom_xsv_metaschema_fail_rejects_invalid_config( make_schema_file: Callable[..., Path], x_xsv_config: dict[str, Any], error_match: str, + via_file: bool, ) -> None: """A schema whose x-xsv-config block violates the custom metaschema raises SchemaError.""" register_xsv_validator() - validator_cls = jsonschema.validators.validator_for({"$schema": CUSTOM_META_SCHEMA_URI}) schema = { "$schema": CUSTOM_META_SCHEMA_URI, "required": ["a"], "x-xsv-config": x_xsv_config, } + if via_file: + schema_path = make_schema_file(schema) + check_fn: Callable[[], object] = partial(validate_xsv_jsonschema, schema_path) + else: + validator_cls = jsonschema.validators.validator_for({"$schema": CUSTOM_META_SCHEMA_URI}) + check_fn = partial(validator_cls.check_schema, schema) + with pytest.raises(jsonschema.exceptions.SchemaError, match=error_match): - validator_cls.check_schema(schema) - with pytest.raises(jsonschema.exceptions.SchemaError, match=error_match): - validate_xsv_jsonschema(make_schema_file(schema)) + check_fn() """generate_header""" diff --git a/tests/docs/test_render_and_sync.py b/tests/docs/test_render_and_sync.py index 95a589b1..f6dade2c 100644 --- a/tests/docs/test_render_and_sync.py +++ b/tests/docs/test_render_and_sync.py @@ -39,18 +39,46 @@ EXPECTED_TEMPLATE_COUNT: Final[int] = 15 EXPECTED_MACRO_STUB_COUNT: Final[int] = 13 +TEMPLATE_FILES: Final[tuple[str, ...]] = tuple( + sorted(p.relative_to(TEMPLATES_DIR).as_posix() for p in TEMPLATES_DIR.rglob("*.md.j2")) +) +PLANNED_DOCS: Final[dict[Path, str]] = build_generated_docs_plan() +PLANNED_MD_STUBS: Final[tuple[tuple[Path, str], ...]] = tuple( + (rel_path, text) for rel_path, text in PLANNED_DOCS.items() if rel_path.suffix == ".md" +) + + +@pytest.fixture(scope="module") +def staged_generated_docs(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Materialize the planned autogenerated docs once for macro stub rendering assertions. + + :param tmp_path_factory: Pytest session temporary directory factory. + :type tmp_path_factory: pytest.TempPathFactory + :return: Directory containing materialized autogenerated documentation files. + :rtype: Path + """ + staged_docs = tmp_path_factory.mktemp("staged_docs") + sync_generated_docs(DocsGenerationPaths(docs_dir=staged_docs)) + return staged_docs + + +@pytest.mark.parametrize( + "rel_template", + TEMPLATE_FILES, + ids=[rel.removesuffix(".md.j2").replace("/", "-") for rel in TEMPLATE_FILES], +) +def test_jinja_template_compiles_pass(rel_template: str) -> None: + """Verify each .md.j2 template compiles without error under StrictUndefined.""" + env = create_jinja_env() + compiled = env.get_template(rel_template) + assert compiled.name == rel_template def test_jinja_templates_inventory_and_strict_macro_registration_pass( monkeypatch: pytest.MonkeyPatch, ) -> None: - """Verify all 15 .md.j2 templates exist, compile without error, and register macros without mutating docs/.""" - template_files = sorted(p.relative_to(TEMPLATES_DIR).as_posix() for p in TEMPLATES_DIR.rglob("*.md.j2")) - assert len(template_files) == EXPECTED_TEMPLATE_COUNT - env = create_jinja_env() - for rel_template in template_files: - compiled = env.get_template(rel_template) - assert compiled.name == rel_template + """Verify all 15 .md.j2 templates exist and register macros without mutating docs/.""" + assert len(TEMPLATE_FILES) == EXPECTED_TEMPLATE_COUNT before_docs_entries = sorted(p.relative_to(DOCS_DIR).as_posix() for p in DOCS_DIR.rglob("*")) for mod_key in [k for k in sys.modules if k == "docs_macros" or k.startswith("docs_macros.")]: @@ -71,22 +99,28 @@ def test_jinja_templates_inventory_and_strict_macro_registration_pass( assert after_docs_entries == before_docs_entries -def test_all_macro_stub_pages_render_via_zensical_loader_pass(tmp_path: Path) -> None: - """Verify all 13 planned render_macros: true stub pages render non-empty Markdown with H1 headings.""" - plan = build_generated_docs_plan() - md_stubs = {rel_path: text for rel_path, text in plan.items() if rel_path.suffix == ".md"} - assert len(md_stubs) == EXPECTED_MACRO_STUB_COUNT +def test_generated_docs_plan_and_sync_inventory_pass(tmp_path: Path) -> None: + """Verify planned Markdown stub count and sync_generated_docs expected path count match.""" + assert len(PLANNED_MD_STUBS) == EXPECTED_MACRO_STUB_COUNT + sync_res = sync_generated_docs(DocsGenerationPaths(docs_dir=tmp_path / "docs")) + assert len(sync_res.expected_paths) == len(PLANNED_DOCS) - staged_docs = tmp_path / "docs" - sync_res = sync_generated_docs(DocsGenerationPaths(docs_dir=staged_docs)) - assert len(sync_res.expected_paths) == len(plan) - for rel_path, stub_text in md_stubs.items(): - assert FRONT_MATTER_RE.match(stub_text) is not None - rendered = materialize_markdown_text(stub_text) - assert rendered.strip().startswith("# "), f"{rel_path.name} did not render an H1 title" - assert "{{" not in rendered or "```" in rendered - assert materialize_markdown_file(staged_docs / rel_path) == rendered +@pytest.mark.parametrize( + ("rel_path", "stub_text"), + [pytest.param(rel_path, stub_text, id=rel_path.stem) for rel_path, stub_text in PLANNED_MD_STUBS], +) +def test_macro_stub_page_renders_via_zensical_loader_pass( + staged_generated_docs: Path, + rel_path: Path, + stub_text: str, +) -> None: + """Verify each planned render_macros: true stub page renders non-empty Markdown with an H1 heading.""" + assert FRONT_MATTER_RE.match(stub_text) is not None + rendered = materialize_markdown_text(stub_text) + assert rendered.strip().startswith("# "), f"{rel_path.name} did not render an H1 title" + assert "{{" not in rendered or "```" in rendered + assert materialize_markdown_file(staged_generated_docs / rel_path) == rendered @pytest.mark.parametrize( @@ -199,11 +233,8 @@ def _stage_synthetic_docs_tree(tmp_path: Path) -> DocsGenerationPaths: return paths -def test_sync_generated_docs_cli_writes_and_prunes_stale_files( - tmp_path: Path, - capsys: pytest.CaptureFixture[str], -) -> None: - """Verify docs_macros CLI --check and sync create missing stubs/schemas and prune stale ones.""" +def test_sync_and_clean_generated_docs_pass_missing_directories(tmp_path: Path) -> None: + """Verify discover_spec_names, clean_generated_docs, and sync_generated_docs handle missing directories.""" empty_paths = DocsGenerationPaths( docs_dir=tmp_path / "empty_docs", converter_specs_root=tmp_path / "missing_conv", @@ -215,6 +246,12 @@ def test_sync_generated_docs_cli_writes_and_prunes_stale_files( assert clean_generated_docs(empty_paths) == () assert sync_generated_docs(empty_paths, include_static_pages=False).expected_paths == () + +def test_sync_generated_docs_cli_writes_and_prunes_stale_files( + tmp_path: Path, + capsys: pytest.CaptureFixture[str], +) -> None: + """Verify docs_macros CLI --check and sync create missing stubs/schemas and prune stale ones.""" paths = _stage_synthetic_docs_tree(tmp_path) stale_conv_stub = paths.docs_dir / "legacy_to_pyarrow_specification.md" stale_conv_stub.write_text(render_macro_stub('render_converter_spec("legacy_to_pyarrow")'), encoding="utf-8") diff --git a/tests/docs/test_specs_context.py b/tests/docs/test_specs_context.py index 7e47ae9c..162816d2 100644 --- a/tests/docs/test_specs_context.py +++ b/tests/docs/test_specs_context.py @@ -199,13 +199,10 @@ def test_render_error_section_rule_style_and_expandable_fixtures_pass(kind: str, assert f"`{param_key}={param_repr}`" in rendered -def test_render_error_section_boundary_cases_pass() -> None: - """Verify partials/errors_section.md.j2 handles empty fixture lists and empty fixture descriptions.""" - template = create_jinja_env().get_template("partials/errors_section.md.j2") - section = ErrorSectionContext( - base_class_name="SampleConversionError", - parent_type_name="ValueError", - rows=( +@pytest.mark.parametrize( + ("row", "expected_snippet"), + [ + pytest.param( ErrorRowContext( code="E1", anchor="e1", @@ -214,6 +211,10 @@ def test_render_error_section_boundary_cases_pass() -> None: reason="Used to test empty fixture fallback.", fixtures=(), ), + " *Fixtures:* —", + id="empty-fixtures", + ), + pytest.param( ErrorRowContext( code="E2", anchor="e2", @@ -230,14 +231,27 @@ def test_render_error_section_boundary_cases_pass() -> None: ), ), ), + ( + " - [`bad.xml`](https://example.org/bad.xml) (`xml_tag='entry'`, `table_name='entries'`)\n\n" + ' ??? abstract "bad.xml"\n\n' + " ```xml\n" + " \n" + " ```" + ), + id="empty-description-with-params", ), + ], +) +def test_render_error_section_boundary_cases_pass( + row: ErrorRowContext, + expected_snippet: str, +) -> None: + """Verify partials/errors_section.md.j2 handles empty fixture lists and empty fixture descriptions.""" + template = create_jinja_env().get_template("partials/errors_section.md.j2") + section = ErrorSectionContext( + base_class_name="SampleConversionError", + parent_type_name="ValueError", + rows=(row,), ) rendered = template.render(error_section=section) - assert " *Fixtures:* —" in rendered - assert ( - " - [`bad.xml`](https://example.org/bad.xml) (`xml_tag='entry'`, `table_name='entries'`)\n\n" - ' ??? abstract "bad.xml"\n\n' - " ```xml\n" - " \n" - " ```" - ) in rendered + assert expected_snippet in rendered