From a5869da7ea0c12845088d8c79e134e38201d6e48 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Sat, 21 Nov 2026 07:47:21 -0800 Subject: [PATCH 1/2] adding tests to improve breadth/depth/quality of test coverage --- .gitignore | 1 + pyproject.toml | 2 + tests/cdm_data_loaders/cli/test_gz.py | 11 + .../cli/test_ncbi_ftp_promote.py | 39 +- .../cli/test_ncbi_ftp_sync.py | 46 + .../cli/test_polaris_schema_definitions.py | 120 +++ .../cli/test_pyiceberg_to_jsonschema.py | 61 +- tests/cdm_data_loaders/coercion/test_xml.py | 177 +++- .../converters/core/test_inference.py | 159 +++- .../converters/emitters/test_dlt.py | 100 +++ .../converters/emitters/test_linkml.py | 121 ++- .../test_pyarrow_codegen_collector.py | 208 +++++ .../facades/test_jsonschema_to_pyarrow.py | 30 + .../converters/sources/test_iceberg.py | 72 ++ .../converters/sources/test_pyarrow_reader.py | 291 +++++++ .../sources/test_pydantic_annotations.py | 279 ++++++ .../sources/test_pydantic_model_synthesis.py | 374 ++++++++ .../sources/test_pydantic_reader_unit.py | 132 +++ .../sources/test_pydantic_unions.py | 281 ++++++ .../sources/test_strict_combinators.py | 187 ++++ .../sources/test_strict_json_schema.py | 240 ++++++ .../converters/sources/test_xsd.py | 187 +++- tests/cdm_data_loaders/core/test_buffer.py | 65 +- tests/cdm_data_loaders/core/test_fields.py | 7 +- .../cdm_data_loaders/core/test_file_items.py | 41 +- .../cdm_data_loaders/core/test_filesystem.py | 43 +- .../core/test_md5_preprocessor.py | 29 +- .../core/test_pipeline_catalog.py | 329 ++++++++ tests/cdm_data_loaders/core/test_reporting.py | 41 + .../core/test_settings_check_aliases.py | 10 +- .../cdm_data_loaders/core/test_transforms.py | 384 ++++++++- .../datasets/ncbi_ftp/test_assembly.py | 59 +- .../datasets/ncbi_ftp/test_manifest.py | 101 +++ tests/cdm_data_loaders/io/test_clients.py | 46 +- .../cdm_data_loaders/parsers/test_checkm2.py | 17 +- .../parsers/test_uniprot_metalink.py | 60 +- .../all_the_bacteria/test_all_the_bacteria.py | 28 + .../pipelines/jsonlines/test_base.py | 45 + .../jsonlines/test_extract_pipeline.py | 21 +- .../jsonlines/test_pydantic_pipeline.py | 92 +- .../pipelines/jsonlines/test_routing.py | 66 ++ .../pipelines/jsonlines/test_settings.py | 42 +- .../cdm_data_loaders/pipelines/test_runner.py | 16 +- .../pipelines/test_xml_common.py | 132 +++ .../xmltodict/test_xmltodict_pipeline.py | 19 +- .../pipelines/xmltopyarrow/test_processors.py | 78 +- .../test_xmltopyarrow_pipeline.py | 15 + .../test_xmltopyarrow_settings.py | 119 +++ .../pipelines/xsv/test_xsv_pipeline.py | 87 ++ .../readers/xml/test_xml_stream_xml_file.py | 119 ++- .../readers/xsv/test_duckdb_reader.py | 51 ++ .../cdm_data_loaders/readers/xsv/test_qsv.py | 59 ++ .../utils/iceberg/test_cleanup.py | 214 ++++- .../utils/test_arrow_schema.py | 2 +- tests/cdm_data_loaders/utils/test_logging.py | 17 + tests/cdm_data_loaders/utils/test_progress.py | 13 +- .../validation/jsonschema/test_dereference.py | 129 +++ .../validation/jsonschema/test_metaschema.py | 73 +- .../validation/xsd/test_particles.py | 187 ++++ .../validation/xsd/test_scalars.py | 52 ++ .../validation/xsd/test_traversal.py | 798 ++++++++++++++++++ uv.lock | 574 ++++++++++++- 62 files changed, 7278 insertions(+), 120 deletions(-) create mode 100644 tests/cdm_data_loaders/converters/emitters/test_pyarrow_codegen_collector.py create mode 100644 tests/cdm_data_loaders/converters/sources/test_pyarrow_reader.py create mode 100644 tests/cdm_data_loaders/converters/sources/test_pydantic_annotations.py create mode 100644 tests/cdm_data_loaders/converters/sources/test_pydantic_model_synthesis.py create mode 100644 tests/cdm_data_loaders/converters/sources/test_pydantic_reader_unit.py create mode 100644 tests/cdm_data_loaders/converters/sources/test_pydantic_unions.py create mode 100644 tests/cdm_data_loaders/converters/sources/test_strict_combinators.py create mode 100644 tests/cdm_data_loaders/converters/sources/test_strict_json_schema.py create mode 100644 tests/cdm_data_loaders/pipelines/test_xml_common.py diff --git a/.gitignore b/.gitignore index 92920303..9aa95f9f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ not_for_github/ cdl_skill/ +mutants/ .DS_Store mutants/ diff --git a/pyproject.toml b/pyproject.toml index 43914c81..cb029d92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -83,6 +83,7 @@ models = [ ] mutants = [ "mutmut>=3.8.1", + "mutmut-mcp", ] docs = [ "griffe-pydantic>=1.3.2", @@ -287,4 +288,5 @@ DLT_PROJECT_DIR = "/tmp/dlt" [tool.uv.sources] cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" } cdm-task-service-client = { git = "https://github.com/kbase/cdm-task-service-client.git" } +mutmut-mcp = { git = "https://github.com/wdm0006/mutmut-mcp" } # berdl-notebook-utils = { git = "https://github.com/KBaseDataLakehouse/spark_notebook.git", subdirectory = "notebook_utils" } diff --git a/tests/cdm_data_loaders/cli/test_gz.py b/tests/cdm_data_loaders/cli/test_gz.py index 8dafee2f..4e1bfe6b 100644 --- a/tests/cdm_data_loaders/cli/test_gz.py +++ b/tests/cdm_data_loaders/cli/test_gz.py @@ -69,6 +69,17 @@ def test_cli_directory_with_glob(tmp_path: Path, source_flag: str, glob_flag: st assert not (sub_dir / "b.log.gz").exists() +def test_main_directory_default_glob(tmp_path: Path) -> None: + """Compress all files in a directory using ``*`` when ``--file-glob`` is omitted.""" + sub_dir = tmp_path / "sub" + sub_dir.mkdir() + for name in ("a.txt", "b.log"): + (sub_dir / name).write_text(name, encoding="utf-8") + main(["--source", str(sub_dir)]) + assert (sub_dir / "a.txt.gz").exists() + assert (sub_dir / "b.log.gz").exists() + + def test_cli_main_nonexistent_source(tmp_path: Path) -> None: """Raise ``RuntimeError`` when ``--source`` points to a missing path.""" missing = tmp_path / "doesnotexist" diff --git a/tests/cdm_data_loaders/cli/test_ncbi_ftp_promote.py b/tests/cdm_data_loaders/cli/test_ncbi_ftp_promote.py index 31c89b03..8420a41f 100644 --- a/tests/cdm_data_loaders/cli/test_ncbi_ftp_promote.py +++ b/tests/cdm_data_loaders/cli/test_ncbi_ftp_promote.py @@ -102,12 +102,6 @@ "dry_run": False, } -MOCK_REPORT_WITH_FAILURES: Final[dict[str, Any]] = { - **MOCK_REPORT_SUCCESS, - "promoted": 3, - "failed": 2, -} - def make_settings(**kwargs: str | int | bool | Path | PurePosixPath | None) -> PromoteSettings: """Build a validated ``PromoteSettings`` instance with a default ``staging_path``. @@ -212,28 +206,31 @@ def test_run_promote_calls_promote_from_s3_with_correct_args(tmp_path: Path) -> ) -def test_run_promote_no_error_on_zero_failures() -> None: - """run_promote does not raise when promote_from_s3 reports zero failures.""" - config = make_settings() - with patch( - "cdm_data_loaders.cli.ncbi_ftp_promote.promote_from_s3", - return_value=MOCK_REPORT_SUCCESS, - ): - run_promote(config) - - -def test_run_promote_logs_error_on_failures(caplog: pytest.LogCaptureFixture) -> None: - """run_promote logs an error without raising when promote_from_s3 reports failures.""" +@pytest.mark.parametrize( + ("failed_count", "expected_messages"), + [ + pytest.param(0, [], id="zero-failures"), + pytest.param(1, ["Promote completed with 1 failures"], id="single-failure"), + pytest.param(2, ["Promote completed with 2 failures"], id="multiple-failures"), + ], +) +def test_run_promote_logs_error_only_when_failures_positive( + caplog: pytest.LogCaptureFixture, + failed_count: int, + expected_messages: list[str], +) -> None: + """run_promote logs an error without raising if and only if promote_from_s3 reports failures > 0.""" config = make_settings() + report = {**MOCK_REPORT_SUCCESS, "failed": failed_count} with ( patch( "cdm_data_loaders.cli.ncbi_ftp_promote.promote_from_s3", - return_value=MOCK_REPORT_WITH_FAILURES, + return_value=report, ), - caplog.at_level(logging.ERROR), + caplog.at_level(logging.ERROR, logger="cdm_data_loaders.cli.ncbi_ftp_promote"), ): run_promote(config) - assert "Promote completed with 2 failures" in caplog.text + assert caplog.messages == expected_messages def test_run_promote_dry_run_forwarded() -> None: diff --git a/tests/cdm_data_loaders/cli/test_ncbi_ftp_sync.py b/tests/cdm_data_loaders/cli/test_ncbi_ftp_sync.py index dbbce73a..a0d0e6b6 100644 --- a/tests/cdm_data_loaders/cli/test_ncbi_ftp_sync.py +++ b/tests/cdm_data_loaders/cli/test_ncbi_ftp_sync.py @@ -1,6 +1,8 @@ """Tests for cli.ncbi_ftp_sync — settings and CLI shortcuts.""" +import logging from pathlib import Path +from unittest.mock import patch import pytest from pydantic import ValidationError @@ -8,6 +10,7 @@ from cdm_data_loaders.cli.ncbi_ftp_sync import ( DownloadSettings, + run_download, ) from cdm_data_loaders.core.fields import INPUT_MOUNT, OUTPUT_MOUNT from cdm_data_loaders.datasets.ncbi_ftp.constants import FTP_HOST @@ -105,3 +108,46 @@ def test_settings_threads_boundaries_accepted(threads: int) -> None: """Boundary thread counts (1 and 32) are accepted.""" s = DownloadSettings(threads=threads) # pyright: ignore[reportCallIssue] assert s.threads == threads + + +@pytest.mark.parametrize( + ("failed_count", "expected_messages"), + [ + pytest.param(0, [], id="no-failures"), + pytest.param(1, ["Download completed with 1 failures"], id="single-failure"), + pytest.param(2, ["Download completed with 2 failures"], id="multiple-failures"), + ], +) +def test_run_download_forwards_settings_and_logs_failures( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + failed_count: int, + expected_messages: list[str], +) -> None: + """run_download forwards all DownloadSettings fields to download_batch and logs failures > 0.""" + manifest = tmp_path / "manifest.txt" + out_dir = tmp_path / "out" + settings = DownloadSettings( + manifest=manifest, + output_dir=out_dir, + threads=CUSTOM_THREADS, + ftp_host="ftp.example.org", + limit=CUSTOM_LIMIT, + ) + with ( + patch( + "cdm_data_loaders.cli.ncbi_ftp_sync.download_batch", + return_value={"failed": failed_count}, + ) as mock_batch, + caplog.at_level(logging.ERROR, logger="cdm_data_loaders.cli.ncbi_ftp_sync"), + ): + run_download(settings) + + mock_batch.assert_called_once_with( + manifest_path=manifest, + output_dir=out_dir, + threads=CUSTOM_THREADS, + ftp_host="ftp.example.org", + limit=CUSTOM_LIMIT, + ) + assert caplog.messages == expected_messages diff --git a/tests/cdm_data_loaders/cli/test_polaris_schema_definitions.py b/tests/cdm_data_loaders/cli/test_polaris_schema_definitions.py index bd5458cc..d98a57db 100644 --- a/tests/cdm_data_loaders/cli/test_polaris_schema_definitions.py +++ b/tests/cdm_data_loaders/cli/test_polaris_schema_definitions.py @@ -1,9 +1,12 @@ """Unit and integration tests for cdm_data_loaders.cli.polaris_schema_definitions.""" import json +import logging from collections.abc import Generator +from contextlib import contextmanager from pathlib import Path from typing import Any, Final +from unittest.mock import patch import dlt import pyarrow as pa @@ -16,7 +19,9 @@ from pyiceberg.utils.config import Config from cdm_data_loaders.cli.polaris_schema_definitions import ( + PolarisSchemaDefinitionsSettings, load_pyarrow_schema, + run_from_settings, ) from cdm_data_loaders.cli.polaris_schema_definitions import ( main as cli_main, @@ -722,3 +727,118 @@ def test_cli_main_applies_definitions_and_prints_json_report( updated_tbl = verify_cat.load_table((NAMESPACE, "id_mapping")) assert updated_tbl.properties["comment"] == "Standardised mapping data from UniProt" assert collect_iceberg_docs(updated_tbl.schema()) == parsed_stdout["updated_fields"] + + +def test_load_pyarrow_schema_blank_description_returns_none(tmp_path: Path) -> None: + """load_pyarrow_schema produces no metadata definition for whitespace-only descriptions.""" + json_path = tmp_path / "blank_desc.json" + json_path.write_text( + json.dumps( + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "description": " ", + "properties": {"id": {"type": "string"}}, + } + ), + encoding="utf-8", + ) + loaded = load_pyarrow_schema(str(json_path)) + assert extract_metadata_definition(loaded.metadata) is None + + +@pytest.mark.parametrize( + ("dry_run", "expected_verb"), + [ + pytest.param(True, "Planned", id="dry-run"), + pytest.param(False, "Applied", id="apply"), + ], +) +def test_run_from_settings_forwards_all_options_and_logs( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, + *, + dry_run: bool, + expected_verb: str, +) -> None: + """run_from_settings forwards catalog and sync options and logs Planned or Applied.""" + json_path = tmp_path / "schema.json" + json_path.write_text( + json.dumps( + { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "properties": {"id": {"type": "string"}}, + } + ), + encoding="utf-8", + ) + + captured_cfg: list[PolarisCatalogConfig] = [] + captured_opts: list[DefinitionSyncOptions] = [] + + @contextmanager + def fake_connect(cfg: PolarisCatalogConfig) -> Generator[object]: + captured_cfg.append(cfg) + yield object() + + def fake_apply( + _cat: object, + _table: str, + _schema: pa.Schema, + *, + options: DefinitionSyncOptions, + ) -> SchemaDefinitionsReport: + captured_opts.append(options) + return SchemaDefinitionsReport( + table_identifier=("ns", "tbl"), + dry_run=options.dry_run, + table_comment_applied="desc", + updated_fields={"a": "b"}, + unchanged_fields=["c"], + ) + + bearer_val = "tok" + settings = PolarisSchemaDefinitionsSettings( + catalog="mycat", + table="ns.tbl", + schema_source=str(json_path), + table_description="override desc", + uri="http://polaris:8181/api/catalog", + warehouse="wh", + credential="cid:csec", + token=bearer_val, + scope="ROLE:ALL", + metadata_keys=("doc",), + normalize_root_names=False, + strict=True, + dry_run=dry_run, + ) + with ( + patch("cdm_data_loaders.cli.polaris_schema_definitions.connect_polaris_catalog", fake_connect), + patch("cdm_data_loaders.cli.polaris_schema_definitions.apply_schema_definitions", fake_apply), + caplog.at_level(logging.INFO, logger="cdm_data_loaders.cli.polaris_schema_definitions"), + ): + report = run_from_settings(settings) + + assert report.dry_run is dry_run + assert captured_cfg == [ + PolarisCatalogConfig( + catalog="mycat", + uri="http://polaris:8181/api/catalog", + warehouse="wh", + credential="cid:csec", + token=bearer_val, + scope="ROLE:ALL", + ) + ] + assert captured_opts == [ + DefinitionSyncOptions( + table_description="override desc", + metadata_keys=("doc",), + normalize_root_names=False, + strict=True, + dry_run=dry_run, + ) + ] + assert f"{expected_verb} definitions on ns.tbl" in caplog.text diff --git a/tests/cdm_data_loaders/cli/test_pyiceberg_to_jsonschema.py b/tests/cdm_data_loaders/cli/test_pyiceberg_to_jsonschema.py index c52e7d30..9ce29d86 100644 --- a/tests/cdm_data_loaders/cli/test_pyiceberg_to_jsonschema.py +++ b/tests/cdm_data_loaders/cli/test_pyiceberg_to_jsonschema.py @@ -17,6 +17,7 @@ from linkml_runtime.linkml_model.meta import SchemaDefinition from linkml_runtime.loaders import yaml_loader from pyiceberg.catalog import Catalog +from pyiceberg.catalog.sql import SqlCatalog from pyiceberg.schema import Schema from pyiceberg.types import BinaryType, NestedField @@ -47,17 +48,17 @@ class CatalogDumpTarget: settings_cls: type[IcebergCatalogDumpSettings] dump_fn: Callable[[Any], None] suffix: str - create_bad_table: Callable[[Catalog, Schema], None] + create_bad_table: Callable[[Catalog, Schema, str], None] assert_table_written: Callable[[Path, str, bool], None] -def _create_bad_jsonschema_table(catalog: Catalog, _simple_schema: Schema) -> None: +def _create_bad_jsonschema_table(catalog: Catalog, _simple_schema: Schema, table_name: str) -> None: unsupported_schema = Schema(NestedField(1, "value", BinaryType(), required=True)) - catalog.create_table(("ns", "bad"), schema=unsupported_schema) + catalog.create_table(("ns", table_name), schema=unsupported_schema) -def _create_bad_linkml_table(catalog: Catalog, simple_schema: Schema) -> None: - bad_table = catalog.create_table(("ns", "bad"), schema=simple_schema) +def _create_bad_linkml_table(catalog: Catalog, simple_schema: Schema, table_name: str) -> None: + bad_table = catalog.create_table(("ns", table_name), schema=simple_schema) bad_table.io.delete(bad_table.metadata_location) @@ -98,7 +99,12 @@ def _assert_linkml_written(path: Path, table_name: str, group_by_namespace: bool @pytest.mark.parametrize("group_by_namespace", [False, True], ids=["per-table", "per-namespace"]) def test_end_to_end_dump_catalog_schemas_writes_valid_docs( - catalog: Catalog, catalog_env: Path, simple_schema: Schema, wide_schema: Schema, group_by_namespace: bool + catalog: Catalog, + catalog_env: Path, + simple_schema: Schema, + wide_schema: Schema, + caplog: pytest.LogCaptureFixture, + group_by_namespace: bool, ) -> None: """dump_catalog_schemas loads the catalog by name and writes valid schema docs, per table or per namespace.""" catalog.create_namespace("ns") @@ -109,12 +115,18 @@ def test_end_to_end_dump_catalog_schemas_writes_valid_docs( settings = IcebergToJsonSchemaSettings( # pyright: ignore[reportCallIssue] catalog=CATALOG_NAME, output_dir=str(out_dir), group_by_namespace=group_by_namespace ) - dump_catalog_schemas(settings) + with caplog.at_level(logging.INFO, logger="cdm_data_loaders.cli.pyiceberg_to_jsonschema"): + dump_catalog_schemas(settings) + + assert caplog.messages[0] == f"Dumping schemas for 1 namespace(s) from catalog {CATALOG_NAME!r}" files = sorted(out_dir.iterdir()) assert [f.name for f in files] == ( ["ns.schema.json"] if group_by_namespace else ["ns.simple.schema.json", "ns.wide.schema.json"] ) + for file_path in files: + raw_text = file_path.read_text(encoding="utf-8") + assert raw_text == json.dumps(json.loads(raw_text), indent=2) if group_by_namespace: namespace_doc = json.loads(files[0].read_text()) @@ -205,9 +217,10 @@ def test_dump_catalog_schemas_pass_skips_failing_table_and_logs( group_by_namespace: bool, include_good: bool, ) -> None: - """A table that fails to convert is logged and skipped; other tables still succeed.""" + """Tables that fail to convert are logged, counted, and skipped; other tables still succeed.""" catalog.create_namespace("ns") - target.create_bad_table(catalog, simple_schema) + target.create_bad_table(catalog, simple_schema, "bad_a") + target.create_bad_table(catalog, simple_schema, "bad_b") if include_good: catalog.create_table(("ns", "good"), schema=simple_schema) @@ -215,15 +228,18 @@ def test_dump_catalog_schemas_pass_skips_failing_table_and_logs( settings = target.settings_cls( # pyright: ignore[reportCallIssue] catalog=CATALOG_NAME, output_dir=str(out_dir), group_by_namespace=group_by_namespace ) - with caplog.at_level(logging.WARNING): + with caplog.at_level(logging.WARNING, logger=target.dump_fn.__module__): target.dump_fn(settings) expected_files = [f"ns{target.suffix}" if group_by_namespace else f"ns.good{target.suffix}"] if include_good else [] assert [f.name for f in sorted(out_dir.iterdir())] == expected_files if include_good: target.assert_table_written(out_dir / expected_files[0], "good", group_by_namespace) - assert "Failed to convert table ('ns', 'bad'); skipping" in caplog.text - assert "Completed with 1 table(s) skipped due to conversion failures" in caplog.text + assert caplog.messages == [ + "Failed to convert table ('ns', 'bad_a'); skipping", + "Failed to convert table ('ns', 'bad_b'); skipping", + "Completed with 2 table(s) skipped due to conversion failures", + ] @pytest.mark.parametrize("target", CATALOG_DUMP_TARGETS) @@ -283,6 +299,27 @@ def test_dump_catalog_schemas_pass_namespace_isolation( assert first["x-iceberg"]["generated_at"] == second["x-iceberg"]["generated_at"] +def test_dump_catalog_schemas_pass_multilevel_namespace( + catalog: Catalog, catalog_env: Path, simple_schema: Schema, monkeypatch: pytest.MonkeyPatch +) -> None: + """Multi-level namespace tuples are dot-joined for grouped filenames and schema metadata.""" + catalog.create_namespace(("parent", "child")) + catalog.create_table(("parent", "child", "record"), schema=simple_schema) + monkeypatch.setattr(SqlCatalog, "list_namespaces", lambda _self, _namespace=(): [("parent", "child")]) + + out_dir = catalog_env / "schemas" + settings = IcebergToJsonSchemaSettings( # pyright: ignore[reportCallIssue] + catalog=CATALOG_NAME, output_dir=str(out_dir), group_by_namespace=True + ) + dump_catalog_schemas(settings) + + assert [file.name for file in out_dir.iterdir()] == ["parent.child.schema.json"] + document = json.loads((out_dir / "parent.child.schema.json").read_text(encoding="utf-8")) + assert document["$id"] == "urn:iceberg:parent.child" + assert document["title"] == "parent.child" + assert set(document["$defs"]) == {"record"} + + @pytest.mark.parametrize("target", CATALOG_DUMP_TARGETS) @pytest.mark.parametrize( ("arguments", "expected"), diff --git a/tests/cdm_data_loaders/coercion/test_xml.py b/tests/cdm_data_loaders/coercion/test_xml.py index dc7baf26..d0150acd 100644 --- a/tests/cdm_data_loaders/coercion/test_xml.py +++ b/tests/cdm_data_loaders/coercion/test_xml.py @@ -1,8 +1,9 @@ """Unit tests for XML-to-Arrow record coercion helpers.""" +import datetime as dt import functools from pathlib import Path -from typing import Final +from typing import Any, Final import pyarrow as pa import pytest @@ -14,7 +15,10 @@ make_xml_element_converter, make_xml_level_converters, ) -from cdm_data_loaders.converters.exceptions.facades.xsd_to_pyarrow import WildcardAttributeError +from cdm_data_loaders.converters.exceptions.facades.xsd_to_pyarrow import ( + FieldNameCollisionError, + WildcardAttributeError, +) from cdm_data_loaders.schemas.uniprot.schema import SCHEMAS as UNIPROT_SCHEMAS from cdm_data_loaders.validation.xsd.traversal import load_schema from tests.helpers.paths_and_constants import BASE_DIR @@ -106,3 +110,172 @@ def test_xml_coercion_entry_points_fail_invalid_max_recursion_depth( with pytest.raises(exc_cls, match="max_recursion_depth"): call() + + +COERCION_XSD: Final[str] = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + + +@pytest.fixture +def coercion_schema() -> xmlschema.XMLSchema: + """Compile the XSD schema fixture used by convert_xml_element tests.""" + return xmlschema.XMLSchema(COERCION_XSD) + + +@pytest.mark.parametrize( + ("xml_bytes", "xml_tag", "max_recursion_depth", "expected"), + [ + pytest.param( + b"0a1bYWJj2026-01-02Z", + "scalars", + None, + {"hex_val": b"\x0a\x1b", "b64_val": b"abc", "date_val": dt.date(2026, 1, 2)}, + id="scalars", + ), + pytest.param( + ( + b'' + b"" + b'' + b"a b" + b"1 2 3" + b'hi <b> bold tail
' + b'42' + b'' + b'' + b'' + b'x' + b"
" + ), + "root", + 1, + { + "_kind": "top", + "req_str": "", + "nmtokens": ["a", "b"], + "int_list": [1, 2, 3], + "mixed_attr": { + "_id": "m1", + "#text": 'hi <b> bold tail
', + }, + "sc": {"_unit": "kg", "#text": 42}, + "empty_attr": {"_flag": True}, + "child": [None, '{"_kind": "leaf", "req_str": "x"}'], + }, + id="mixed-nil-lists-and-truncated-child", + ), + pytest.param( + b'', + "root", + None, + {"_kind": "k"}, + id="nil-root-with-attribute", + ), + pytest.param( + b'', + "root", + None, + {}, + id="nil-root-empty", + ), + ], +) +def test_convert_xml_element_pass_scalars_mixed_nil_and_lists( + coercion_schema: xmlschema.XMLSchema, + xml_bytes: bytes, + xml_tag: str, + max_recursion_depth: int | None, + expected: dict[str, Any], +) -> None: + """convert_xml_element coerces scalars, xsi:nil elements, mixed inline XML, lists, and truncated recursion.""" + result = ( + convert_xml_element(fromstring(xml_bytes), coercion_schema, xml_tag) + if max_recursion_depth is None + else convert_xml_element( + fromstring(xml_bytes), + coercion_schema, + xml_tag, + max_recursion_depth=max_recursion_depth, + ) + ) + assert result == expected + + +def test_convert_xml_element_fail_attribute_child_collision() -> None: + """convert_xml_element raises FieldNameCollisionError when an attribute and child map to the same name.""" + collision_xsd = xmlschema.XMLSchema( + """ + + + + + + + + + """ + ) + with pytest.raises(FieldNameCollisionError, match="Field name collision"): + convert_xml_element(fromstring(b'<_id>2'), collision_xsd, "root") + + +def test_make_xml_level_converters_pass_nil_leaf_element() -> None: + """make_xml_level_converters returns an empty dict for a nil scalar leaf element.""" + leaf_xsd = xmlschema.XMLSchema( + """ + + + + + + + + """ + ) + note_conv = make_xml_level_converters(leaf_xsd, 1)["note"] + assert note_conv(fromstring(b'')) == {} diff --git a/tests/cdm_data_loaders/converters/core/test_inference.py b/tests/cdm_data_loaders/converters/core/test_inference.py index 0a580044..7bf0cc91 100644 --- a/tests/cdm_data_loaders/converters/core/test_inference.py +++ b/tests/cdm_data_loaders/converters/core/test_inference.py @@ -1,7 +1,7 @@ """Tests for target-independent type inference.""" from decimal import Decimal -from typing import Any +from typing import Any, Final import pytest @@ -15,8 +15,53 @@ infer_implicit_type, is_unconstrained_node, json_type_from_enum, + nodes_same_shape, + unify_scalar_specs, +) +from cdm_data_loaders.converters.core.ir import ( + SPEC_FLOAT64, + SPEC_STRING, + SPEC_TS_UTC, + Field, + NodeHints, + TypedNode, ) -from cdm_data_loaders.converters.core.ir import TypedNode + +DEC_10_2: Final[tuple[str, NodeHints, str]] = ( + "number", + NodeHints(logical_type="decimal", precision=10, scale=2), + "Decimal(10, 2)", +) +DEC_12_4: Final[tuple[str, NodeHints, str]] = ( + "number", + NodeHints(logical_type="decimal", precision=12, scale=4), + "Decimal(12, 4)", +) +DEC_38_0: Final[tuple[str, NodeHints, str]] = ( + "number", + NodeHints(logical_type="decimal", precision=38, scale=0), + "Decimal(38, 0)", +) +LONG_SPEC: Final[tuple[str, NodeHints, str]] = ("integer", NodeHints(logical_type="long", bit_width=64), "int") +TS_TZ_SPEC: Final[tuple[str, NodeHints, str]] = ( + "string", + NodeHints(logical_type="timestamp-with-tz", timezone=True), + "AwareDT", +) +TS_NAIVE_SPEC: Final[tuple[str, NodeHints, str]] = ( + "string", + NodeHints(logical_type="timestamp-without-tz", timezone=False), + "NaiveDT", +) +BOOL_SPEC: Final[tuple[str, NodeHints, str]] = ("boolean", NodeHints(logical_type="boolean"), "bool") + +STR_NODE: Final[TypedNode] = TypedNode("string") +NULLABLE_STR_NODE: Final[TypedNode] = TypedNode("string", nullable=True) +INT_NODE: Final[TypedNode] = TypedNode("integer") +DATE_STR_NODE: Final[TypedNode] = TypedNode("string", hints=NodeHints(logical_type="date")) +ARRAY_STR_NODE: Final[TypedNode] = TypedNode("array", items=STR_NODE) +MAP_STR_INT_NODE: Final[TypedNode] = TypedNode("map", key_type=STR_NODE, value_type=INT_NODE) +OBJ_X_REQ_NODE: Final[TypedNode] = TypedNode("object", properties=(Field("x", STR_NODE, required=True),)) @pytest.mark.parametrize( @@ -178,11 +223,119 @@ def test_decimal_places_pass_scale(value: float | Decimal, expected: int) -> Non ("precision", "scale", "expected"), [ (10, 2, r"^-?\d{1,8}(\.\d{1,2})?$"), + (5, 2, r"^-?\d{1,3}(\.\d{1,2})?$"), (2, 2, r"^-?0(\.\d{1,2})?$"), (10, 0, r"^-?\d{1,10}$"), + (4, 0, r"^-?\d{1,4}$"), ], - ids=["standard", "zero-before-decimal", "no-scale"], + ids=["standard", "small-standard", "zero-before-decimal", "no-scale", "small-no-scale"], ) def test_decimal_pattern_pass_regex(precision: int, scale: int, expected: str) -> None: """Build precise regex patterns for decimal constraints.""" assert decimal_pattern(precision, scale) == expected + + +@pytest.mark.parametrize( + ("specs", "expected"), + [ + pytest.param([DEC_10_2, DEC_10_2], ("number", DEC_10_2[1], "Decimal(10, 2)"), id="identical-decimals"), + pytest.param( + [DEC_10_2, DEC_12_4], + ("number", NodeHints(logical_type="decimal", precision=12, scale=4), "Decimal(10, 2) | Decimal(12, 4)"), + id="widened-decimals", + ), + pytest.param( + [DEC_38_0, DEC_12_4], + (*SPEC_FLOAT64, "Decimal(38, 0) | Decimal(12, 4)"), + id="decimal-precision-overflow", + ), + pytest.param([LONG_SPEC, DEC_10_2], (*SPEC_FLOAT64, "int | Decimal(10, 2)"), id="integer-and-decimal"), + pytest.param([TS_TZ_SPEC, TS_NAIVE_SPEC], (*SPEC_TS_UTC, "AwareDT | NaiveDT"), id="mixed-timestamps"), + pytest.param([LONG_SPEC, BOOL_SPEC], (*SPEC_STRING, "int | bool"), id="incompatible-fallback"), + ], +) +def test_unify_scalar_specs_pass( + specs: list[tuple[str, NodeHints, str]], + expected: tuple[str, NodeHints, str], +) -> None: + """unify_scalar_specs unifies identical specs, decimals, numeric or timestamp groups, and fallbacks.""" + assert unify_scalar_specs(specs) == expected + + +@pytest.mark.parametrize( + ("left", "right", "expected"), + [ + pytest.param(STR_NODE, NULLABLE_STR_NODE, True, id="scalar-ignores-nullability"), + pytest.param(STR_NODE, INT_NODE, False, id="scalar-different-type"), + pytest.param(STR_NODE, DATE_STR_NODE, False, id="scalar-different-hints"), + pytest.param(ARRAY_STR_NODE, TypedNode("array", items=STR_NODE), True, id="array-same-items"), + pytest.param(ARRAY_STR_NODE, TypedNode("array", items=NULLABLE_STR_NODE), False, id="array-item-nullability"), + pytest.param(ARRAY_STR_NODE, TypedNode("array", items=INT_NODE), False, id="array-item-type"), + pytest.param(ARRAY_STR_NODE, TypedNode("array", items=None), False, id="array-missing-items"), + pytest.param(ARRAY_STR_NODE, TypedNode("array", items=(STR_NODE,)), False, id="array-tuple-items"), + pytest.param( + MAP_STR_INT_NODE, + TypedNode("map", key_type=STR_NODE, value_type=INT_NODE), + True, + id="map-same-types", + ), + pytest.param( + MAP_STR_INT_NODE, + TypedNode("map", key_type=None, value_type=INT_NODE), + False, + id="map-missing-key", + ), + pytest.param( + MAP_STR_INT_NODE, + TypedNode("map", key_type=STR_NODE, value_type=None), + False, + id="map-missing-value", + ), + pytest.param( + MAP_STR_INT_NODE, + TypedNode("map", key_type=INT_NODE, value_type=INT_NODE), + False, + id="map-different-key", + ), + pytest.param( + MAP_STR_INT_NODE, + TypedNode("map", key_type=STR_NODE, value_type=STR_NODE), + False, + id="map-different-value", + ), + pytest.param( + MAP_STR_INT_NODE, + TypedNode("map", key_type=STR_NODE, value_type=TypedNode("integer", nullable=True)), + False, + id="map-value-nullability", + ), + pytest.param( + OBJ_X_REQ_NODE, + TypedNode("object", properties=(Field("x", STR_NODE, required=True),)), + True, + id="object-same-properties", + ), + pytest.param(OBJ_X_REQ_NODE, TypedNode("object", properties=()), False, id="object-property-count"), + pytest.param( + OBJ_X_REQ_NODE, + TypedNode("object", properties=(Field("y", STR_NODE, required=True),)), + False, + id="object-property-name", + ), + pytest.param( + OBJ_X_REQ_NODE, + TypedNode("object", properties=(Field("x", STR_NODE, required=False),)), + False, + id="object-property-required", + ), + pytest.param( + OBJ_X_REQ_NODE, + TypedNode("object", properties=(Field("x", INT_NODE, required=True),)), + False, + id="object-property-type", + ), + ], +) +def test_nodes_same_shape_pass(left: TypedNode, right: TypedNode, *, expected: bool) -> None: + """nodes_same_shape compares types, hints, arrays, maps, and objects recursively.""" + assert nodes_same_shape(left, right) is expected diff --git a/tests/cdm_data_loaders/converters/emitters/test_dlt.py b/tests/cdm_data_loaders/converters/emitters/test_dlt.py index f62ef710..e17dfcfd 100644 --- a/tests/cdm_data_loaders/converters/emitters/test_dlt.py +++ b/tests/cdm_data_loaders/converters/emitters/test_dlt.py @@ -539,3 +539,103 @@ def test_emit_pass_native_empty_table_and_new_child( new_child = Field("new", TypedNode("object", properties=(Field("value", TypedNode("string")),))) document = replace(document, root=replace(document.root, properties=(new_child,))) assert DltEmitter(schema_name="source").emit(document)["tables"] == expected_tables + + +def test_emit_pass_json_unions_and_boolean_warnings(caplog: pytest.LogCaptureFixture) -> None: + """DltEmitter approximates multi-type unions, anyOf/oneOf, and boolean schemas with warnings.""" + source = base_object_schema( + properties={ + "null_only": {"type": ["null"]}, + "multi_union": {"type": ["string", "integer"]}, + "nullable_str": {"type": ["string", "null"]}, + "anyof_bool": {"anyOf": [True, {"type": "integer"}]}, + "oneof_typed": {"oneOf": [{"type": "integer"}, {"type": "string"}]}, + "bool_false": False, + }, + ) + doc = JsonSchemaReader().read(source) + with caplog.at_level("WARNING", logger="cdm_data_loaders.converters.emitters.dlt"): + tables = DltEmitter(schema_name="u").emit(doc)["tables"] + cols = tables["u"]["columns"] + assert cols["null_only"]["data_type"] == "text" + assert cols["multi_union"]["data_type"] == "text" + assert cols["nullable_str"]["data_type"] == "text" + assert cols["anyof_bool"]["data_type"] == "text" + assert cols["oneof_typed"]["data_type"] == "bigint" + assert cols["bool_false"]["data_type"] == "text" + + messages = "\n".join(r.getMessage() for r in caplog.records) + assert "Collapsing multi-type union" in messages + assert "Approximating 'anyOf'" in messages + assert "Approximating 'oneOf'" in messages + assert "Boolean schema for property 'bool_false'" in messages + + +def test_emit_pass_column_hints_precision_and_timezone() -> None: + """DltEmitter emits precision and timezone column hints for timestamps, fixed lengths, and bit widths.""" + ts_tz = TypedNode("string", hints=NodeHints(logical_type="timestamp-with-tz", timezone=True)) + ts_no_tz = TypedNode("string", hints=NodeHints(logical_type="timestamp-without-tz", timezone=False)) + fixed_len = TypedNode("string", hints=NodeHints(logical_type="fixed", length=16)) + int_bits = TypedNode("integer", hints=NodeHints(bit_width=32)) + hints_doc = SchemaDocument( + root=TypedNode( + "object", + properties=( + Field("ts_tz", ts_tz), + Field("ts_no_tz", ts_no_tz), + Field("fixed_len", fixed_len), + Field("int_bits", int_bits), + ), + ) + ) + hint_cols = DltEmitter(schema_name="h").emit(hints_doc)["tables"]["h"]["columns"] + ts_precision = 6 + fixed_precision = 16 + int_precision = 32 + assert hint_cols["ts_tz"]["precision"] == ts_precision + assert hint_cols["ts_tz"]["timezone"] is True + assert hint_cols["ts_no_tz"]["precision"] == ts_precision + assert hint_cols["ts_no_tz"]["timezone"] is False + assert hint_cols["fixed_len"]["precision"] == fixed_precision + assert hint_cols["int_bits"]["precision"] == int_precision + + +def test_emit_pass_native_restore_table_and_column_metadata() -> None: + """Native dlt provenance restores table and column metadata, nullable/precision None values, and descriptions.""" + source = { + "root": { + "name": "root", + "description": "old root desc", + "columns": { + "c_named": {"name": "c_named", "data_type": "text", "nullable": True, "precision": None}, + "c_any": {"data_type": "text", "nullable": False, "scale": None, "timezone": None}, + "c_null_none": {"data_type": "bigint", "nullable": False}, + }, + }, + "root__sub": {"name": "root__sub", "parent": "root", "description": "sub desc", "columns": {}}, + } + doc = DltReader().read(source)["root"] + # drop root description and override field nullability/type to exercise native metadata restoration + root_props = list(doc.root.properties) + c_named, c_any, c_null_none, sub_field = root_props + c_any_mod = replace(c_any, node=TypedNode("any", nullable=True, extensions=c_any.node.extensions)) + c_null_none_mod = replace(c_null_none, node=replace(c_null_none.node, nullable=None)) + doc_mod = replace( + doc, + root=replace( + doc.root, + annotations={}, + properties=(c_named, c_any_mod, c_null_none_mod, sub_field), + ), + ) + tables = DltEmitter(schema_name="s").emit(doc_mod)["tables"] + assert "description" not in tables["root"] + assert tables["root"]["name"] == "root" + assert tables["root"]["columns"]["c_named"]["name"] == "c_named" + assert tables["root"]["columns"]["c_named"]["precision"] is None + assert tables["root"]["columns"]["c_any"]["data_type"] is None + assert tables["root"]["columns"]["c_any"]["scale"] is None + assert tables["root"]["columns"]["c_any"]["timezone"] is None + assert tables["root"]["columns"]["c_null_none"]["nullable"] is None + assert tables["root__sub"]["description"] == "sub desc" + assert tables["root__sub"]["columns"] == {} diff --git a/tests/cdm_data_loaders/converters/emitters/test_linkml.py b/tests/cdm_data_loaders/converters/emitters/test_linkml.py index 83fc0698..6778f94f 100644 --- a/tests/cdm_data_loaders/converters/emitters/test_linkml.py +++ b/tests/cdm_data_loaders/converters/emitters/test_linkml.py @@ -8,9 +8,13 @@ from linkml_runtime.linkml_model.meta import ClassDefinition, EnumDefinition, SchemaDefinition, SlotDefinition from linkml_runtime.loaders import yaml_loader -from cdm_data_loaders.converters.core.ir import Field, SchemaDocument, TypedNode +from cdm_data_loaders.converters.core.ir import Field, NodeHints, Provenance, SchemaDocument, TypedNode from cdm_data_loaders.converters.emitters.linkml import LinkMLEmitter -from cdm_data_loaders.converters.exceptions.emitters.linkml import LinkMLEmitterError +from cdm_data_loaders.converters.exceptions.emitters.linkml import ( + EmptyLinkMLSlotNameError, + LinkMLEmitterError, + NestedLinkMLMultivaluedError, +) from cdm_data_loaders.converters.sources.json_schema.permissive import JsonSchemaReader @@ -262,3 +266,116 @@ def test_emit_into_fail_preserves_destination(root: TypedNode) -> None: "good": ClassDefinition(name="good"), } assert schema.enums == {} + + +def test_emit_field_fail_blank_name() -> None: + """emit_field rejects whitespace-only slot names with EmptyLinkMLSlotNameError.""" + with pytest.raises(EmptyLinkMLSlotNameError, match="non-empty"): + LinkMLEmitter().emit_field(Field(" ", TypedNode("string"))) + + +def test_emit_field_pass_node_description() -> None: + """emit_field falls back to the node description when the field has no annotation.""" + slot = LinkMLEmitter().emit_field(Field("col", TypedNode("string", annotations={"description": "from node"}))) + assert slot.description == "from node" + + +@pytest.mark.parametrize( + ("node", "error_type", "match"), + [ + pytest.param( + TypedNode("array", items=TypedNode("array", items=TypedNode("string"))), + NestedLinkMLMultivaluedError, + None, + id="nested-array", + ), + pytest.param( + TypedNode("array", items=TypedNode("map", key_type=TypedNode("string"), value_type=TypedNode("string"))), + NestedLinkMLMultivaluedError, + None, + id="array-of-map", + ), + pytest.param( + TypedNode("map", key_type=None, value_type=TypedNode("string")), + LinkMLEmitterError, + "key and value", + id="map-missing-key", + ), + ], +) +def test_emit_node_fail_invalid_multivalued_and_map( + node: TypedNode, + error_type: type[Exception], + match: str | None, +) -> None: + """emit_node rejects nested multivalued nodes and incomplete map nodes.""" + with pytest.raises(error_type, match=match): + LinkMLEmitter().emit_node(node) + + +def test_emit_pass_pyarrow_bounds_fixed_list_and_map() -> None: + """LinkMLEmitter uses document title and emits PyArrow bounds, fixed-size lists, and maps.""" + pa_prov = Provenance("pyarrow", ()) + pa_doc = SchemaDocument( + annotations={"title": "MyTitle"}, + root=TypedNode( + "object", + properties=( + Field( + "u8_list", + TypedNode( + "array", + hints=NodeHints(logical_type="fixed-size-list", length=4), + items=TypedNode( + "integer", + hints=NodeHints(logical_type="uint8", bit_width=8), + provenance=pa_prov, + ), + provenance=pa_prov, + ), + ), + Field( + "i16", + TypedNode( + "integer", + hints=NodeHints(logical_type="int16", bit_width=16), + provenance=pa_prov, + ), + ), + Field( + "pa_map", + TypedNode( + "map", + key_type=TypedNode("string", provenance=pa_prov), + value_type=TypedNode("integer", nullable=False, provenance=pa_prov), + provenance=pa_prov, + ), + ), + ), + ), + ) + out = LinkMLEmitter().emit(pa_doc) + assert out.name == "MyTitle" + assert "MyTitle" in out.classes + attrs = out.classes["MyTitle"].attributes + u8_list = attrs["u8_list"] + expected_len = 4 + assert u8_list.minimum_value is None + assert u8_list.maximum_value is None + assert u8_list.minimum_cardinality == expected_len + assert u8_list.maximum_cardinality == expected_len + min_i16 = -32768 + max_i16 = 32767 + assert attrs["i16"].minimum_value == min_i16 + assert attrs["i16"].maximum_value == max_i16 + map_cls = out.classes["MyTitle_pa_map_map"] + assert map_cls.attributes["key"].required is True + assert map_cls.attributes["value"].required is True + + +def test_emit_into_pass_avoids_enum_name_collision_and_unnamed_root() -> None: + """emit_into avoids collisions with existing enum names and handles unnamed root documents.""" + target = SchemaDefinition(id="urn:test:ns", name="ns", enums={"Root": EnumDefinition(name="Root")}) + LinkMLEmitter().emit_into(SchemaDocument(root=TypedNode("object")), target) + assert "Root_2" in target.classes + assert target.classes["Root_2"].description is None diff --git a/tests/cdm_data_loaders/converters/emitters/test_pyarrow_codegen_collector.py b/tests/cdm_data_loaders/converters/emitters/test_pyarrow_codegen_collector.py new file mode 100644 index 00000000..5d0c915f --- /dev/null +++ b/tests/cdm_data_loaders/converters/emitters/test_pyarrow_codegen_collector.py @@ -0,0 +1,208 @@ +"""Unit tests for cdm_data_loaders.converters.emitters.pyarrow_codegen.collector.""" + +from typing import Final + +import pyarrow as pa +import pytest + +from cdm_data_loaders.converters.core.ir import Field, Provenance, SchemaDocument, TypedNode +from cdm_data_loaders.converters.emitters.pyarrow_codegen.collector import ( + IRComponentCollector, + extract_component_meta, + unwrap_array_element_node, + unwrap_map_value_node, + unwrap_object_node, +) +from cdm_data_loaders.converters.emitters.pyarrow_codegen.naming import RenderedFieldSpec + +PLAIN_OBJECT_NODE: Final[TypedNode] = TypedNode("object") +ELEM_STRING_NODE: Final[TypedNode] = TypedNode("string") +MAP_VALUE_INT_NODE: Final[TypedNode] = TypedNode("integer") +BAD_NAME_OBJECT_NODE: Final[TypedNode] = TypedNode("object", provenance=Provenance("xsd", (), {"type_name": 123})) + + +@pytest.mark.parametrize( + ("node", "expected"), + [ + pytest.param( + PLAIN_OBJECT_NODE, + ((id(PLAIN_OBJECT_NODE), False), None, False), + id="plain-object-falls-back-to-node-id", + ), + pytest.param( + TypedNode( + "object", + provenance=Provenance( + "xsd", + (), + {"component_id": 7, "nillable": True, "type_name": "ItemType", "inline_struct": True}, + ), + ), + ((7, True), "ItemType", True), + id="provenance-metadata-present", + ), + pytest.param( + BAD_NAME_OBJECT_NODE, + ((id(BAD_NAME_OBJECT_NODE), False), None, False), + id="non-string-type-name-ignored", + ), + ], +) +def test_extract_component_meta_variants( + node: TypedNode, + expected: tuple[tuple[int, bool], str | None, bool], +) -> None: + """Verify extract_component_meta extracts component key, type name, and inline flag.""" + assert extract_component_meta(node) == expected + + +@pytest.mark.parametrize( + ("node", "expected"), + [ + pytest.param(None, None, id="none"), + pytest.param(TypedNode("string"), None, id="scalar"), + pytest.param(PLAIN_OBJECT_NODE, PLAIN_OBJECT_NODE, id="direct-object"), + pytest.param(TypedNode("any", one_of=(PLAIN_OBJECT_NODE,)), PLAIN_OBJECT_NODE, id="one-of-object"), + pytest.param(TypedNode("any", any_of=(PLAIN_OBJECT_NODE,)), PLAIN_OBJECT_NODE, id="any-of-object"), + ], +) +def test_unwrap_object_node_variants(node: TypedNode | None, expected: TypedNode | None) -> None: + """Verify unwrap_object_node unwraps direct and combinator-wrapped object nodes.""" + assert unwrap_object_node(node) is expected + + +@pytest.mark.parametrize( + ("node", "expected"), + [ + pytest.param(None, None, id="none"), + pytest.param(TypedNode("array", prefix_items=(ELEM_STRING_NODE,)), ELEM_STRING_NODE, id="prefix-items"), + pytest.param(TypedNode("array", items=(ELEM_STRING_NODE,)), ELEM_STRING_NODE, id="items-tuple"), + pytest.param(TypedNode("array", items=()), None, id="empty-items-tuple"), + pytest.param(TypedNode("array", items=ELEM_STRING_NODE), ELEM_STRING_NODE, id="single-items-node"), + ], +) +def test_unwrap_array_element_node_variants(node: TypedNode | None, expected: TypedNode | None) -> None: + """Verify unwrap_array_element_node extracts the element node from array variants.""" + assert unwrap_array_element_node(node) is expected + + +@pytest.mark.parametrize( + ("node", "expected"), + [ + pytest.param(None, None, id="none"), + pytest.param(TypedNode("map", value_type=MAP_VALUE_INT_NODE), MAP_VALUE_INT_NODE, id="map-value-type"), + pytest.param( + TypedNode("object", pattern_properties={"^k": MAP_VALUE_INT_NODE}), + MAP_VALUE_INT_NODE, + id="pattern-properties", + ), + pytest.param( + TypedNode("object", additional_properties=MAP_VALUE_INT_NODE), + MAP_VALUE_INT_NODE, + id="additional-properties", + ), + pytest.param( + TypedNode("object", additional_properties=TypedNode("any")), + None, + id="additional-properties-any", + ), + pytest.param( + TypedNode("object", additional_properties=TypedNode("never")), + None, + id="additional-properties-never", + ), + ], +) +def test_unwrap_map_value_node_variants(node: TypedNode | None, expected: TypedNode | None) -> None: + """Verify unwrap_map_value_node extracts value nodes for maps and map-like objects.""" + assert unwrap_map_value_node(node) is expected + + +def test_ir_component_collector_collect_document_scalar_leaf_table() -> None: + """Verify collect_document returns a RenderedFieldSpec directly for scalar leaf tables.""" + collector = IRComponentCollector() + scalar_doc = SchemaDocument(root=TypedNode("object", provenance=Provenance("xsd", (), {"scalar_leaf_table": True}))) + + scalar_spec = collector.collect_document("note", scalar_doc, pa.schema([pa.field("#text", pa.string())])) + + assert isinstance(scalar_spec, RenderedFieldSpec) + assert scalar_spec.name == "#text" + + +def test_ir_component_collector_scan_collect_and_assign_names() -> None: + """Verify IRComponentCollector handles truncated IDs, maps, nested lists, collisions, and inline fallback.""" + collector = IRComponentCollector() + + trunc_node = TypedNode("string", provenance=Provenance("xsd", (), {"truncated_component_id": 99})) + leaf_obj = TypedNode( + "object", + properties=(Field("v", TypedNode("integer")),), + provenance=Provenance("json-schema", (), {"component_id": 10}), + ) + leaf_obj_same_key_diff_struct = TypedNode( + "object", + properties=(Field("v", TypedNode("string")),), + provenance=Provenance("json-schema", (), {"component_id": 10}), + ) + cyclic_obj = TypedNode( + "object", + properties=(Field("name", TypedNode("string")),), + provenance=Provenance("xsd", (), {"component_id": 20}), + ) + # self-referential properties via another node sharing component_id=20 + cyclic_holder = TypedNode( + "object", + properties=( + Field("self_ref", cyclic_obj), + Field("trunc", trunc_node), + Field("nested_map", TypedNode("map", value_type=leaf_obj)), + Field("nested_list", TypedNode("array", items=TypedNode("array", items=leaf_obj_same_key_diff_struct))), + Field("empty_obj", TypedNode("object")), + ), + provenance=Provenance("xsd", (), {"component_id": 20}), + ) + doc = SchemaDocument(root=cyclic_holder) + collector.register_document("root", doc) + trunc_id = 99 + assert trunc_id in collector.el_name_types["trunc"] + assert "empty_obj" not in collector.el_name_types + + pa_leaf_int = pa.struct([pa.field("v", pa.int64())]) + pa_leaf_str = pa.struct([pa.field("v", pa.string())]) + pa_schema = pa.schema( + [ + pa.field("self_ref", pa.struct([pa.field("name", pa.string())])), + pa.field("trunc", pa.string()), + pa.field("nested_map", pa.map_(pa.string(), pa_leaf_int)), + pa.field("nested_list", pa.list_(pa.list_(pa_leaf_str))), + pa.field("empty_obj", pa.struct([])), + ] + ) + root_key = collector.collect_document("root", doc, pa_schema) + assert root_key == (20, False) + + # nillable=True on component_id=10 triggers _NILLABLE suffix + nillable_leaf = TypedNode( + "object", + properties=(Field("v", TypedNode("integer")),), + provenance=Provenance("xsd", (), {"component_id": 10, "nillable": True}), + ) + collector.collect_document("nested_map", SchemaDocument(root=nillable_leaf), pa.schema([pa.field("v", pa.int64())])) + + # duplicate root name without parent triggers _2_STRUCT suffix + dup_root = TypedNode( + "object", + properties=(Field("v", TypedNode("integer")),), + provenance=Provenance("xsd", (), {"component_id": 30}), + ) + collector.collect_document("nested_map", SchemaDocument(root=dup_root), pa.schema([pa.field("v", pa.int64())])) + + names = collector.assign_component_names() + assert names[(10, True)] == "NESTED_MAP_NILLABLE_STRUCT" + assert names[(30, False)] == "NESTED_MAP_2_STRUCT" + assert "__COMPONENT_10_0__" in next( + f.type_expr for f in collector.components[(20, False)].fields if f.name == "nested_map" + ) + # differing pa_struct for same json-schema component_id=10 renders inline + nested_list_expr = next(f.type_expr for f in collector.components[(20, False)].fields if f.name == "nested_list") + assert "pa.struct([" in nested_list_expr diff --git a/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py b/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py index d623143f..f3d5681b 100644 --- a/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py +++ b/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py @@ -2,6 +2,7 @@ import functools from collections.abc import Callable +from typing import Any import pyarrow as pa import pytest @@ -15,6 +16,8 @@ ) from cdm_data_loaders.converters.facades.jsonschema_to_pyarrow import ( JSONSchemaToPyArrow, + generate_pyarrow_schema, + resolve_scalar_type, ) from cdm_data_loaders.converters.facades.pydantic_to_pyarrow import PydanticToPyArrow from cdm_data_loaders.converters.facades.xsd_to_pyarrow import XsdToPyArrow @@ -226,3 +229,30 @@ def test_strict_ref_helpers_fail_direct_alias_cycle( error_type=JSONSchemaToPyArrowError, error_by_code=ERROR_BY_CODE, ) + + +@pytest.mark.parametrize( + ("node", "expected"), + [ + pytest.param({"type": "integer"}, (pa.int64(), "integer"), id="integer"), + pytest.param({"type": "string", "format": "date-time"}, (pa.string(), "string (date-time)"), id="date-time"), + pytest.param({"enum": ["a", "b"]}, (pa.string(), "string"), id="string-enum"), + ], +) +def test_resolve_scalar_type_pass(node: dict[str, Any], expected: tuple[pa.DataType, str]) -> None: + """resolve_scalar_type maps scalar JSON Schema nodes to PyArrow types and labels.""" + assert resolve_scalar_type(node) == expected + + +def test_resolve_scalar_type_fail_non_scalar() -> None: + """resolve_scalar_type rejects non-scalar JSON Schema nodes.""" + with pytest.raises(JSONSchemaToPyArrowError): + resolve_scalar_type({"type": "object", "properties": {"a": {"type": "string"}}}) + + +def test_generate_pyarrow_schema_pass() -> None: + """generate_pyarrow_schema converts a JSON Schema object into a PyArrow schema.""" + schema = base_object_schema(properties={"x": {"type": "boolean"}}) + assert generate_pyarrow_schema(schema) == pa.schema( + [pa.field("x", pa.bool_(), metadata={"jsonschema_type": "boolean"})] + ) diff --git a/tests/cdm_data_loaders/converters/sources/test_iceberg.py b/tests/cdm_data_loaders/converters/sources/test_iceberg.py index 26cd1dca..c3409fba 100644 --- a/tests/cdm_data_loaders/converters/sources/test_iceberg.py +++ b/tests/cdm_data_loaders/converters/sources/test_iceberg.py @@ -34,6 +34,7 @@ ) from cdm_data_loaders.converters.core.errors import ConversionError +from cdm_data_loaders.converters.core.ir import TypedNode from cdm_data_loaders.converters.sources.iceberg import IcebergReader, iceberg_value from tests.helpers.iceberg_catalog import make_iceberg_table as make_table @@ -153,6 +154,20 @@ def test_read_pass_empty_schema() -> None: "identifier", id="empty-table-identifier", ), + pytest.param( + functools.partial(IcebergReader().read_table, make_table(("ns", "table"), Schema()), ("", "t")), + "nonempty strings", + id="empty-identifier-part", + ), + pytest.param( + functools.partial( + IcebergReader().read_table, + make_table(("ns", "table"), Schema()), + (123, "t"), # type: ignore[arg-type] + ), + "nonempty strings", + id="non-string-identifier-part", + ), ], ) def test_read_fail_invalid_source_and_identifier( @@ -207,3 +222,60 @@ def test_read_fail_unsupported_type_and_metadata( """Unmapped Iceberg types and arbitrary Python defaults are not guessed.""" with pytest.raises(error_type, match=match): call() + + +def test_read_pass_provenance_and_extensions() -> None: + """IcebergReader.read preserves schema provenance paths, x-iceberg metadata, and closed root additional_properties.""" + schema = Schema( + NestedField(1, "id", LongType(), required=True, doc="primary id"), + NestedField(2, "nested", StructType(NestedField(3, "inner", StringType()))), + schema_id=7, + identifier_field_ids=[1], + ) + doc = IcebergReader().read(schema) + expected_schema_id = 7 + assert doc.extensions["x-iceberg"] == {"schema_id": expected_schema_id, "identifier_field_ids": (1,)} + assert doc.provenance is not None + assert doc.provenance.source == "iceberg" + assert doc.provenance.metadata["schema-id"] == expected_schema_id + assert doc.root.additional_properties == TypedNode(type="never", nullable=False) + id_field, nested_field = doc.root.properties + assert id_field.annotations == {"description": "primary id"} + assert id_field.provenance is not None + assert id_field.provenance.path == ("id",) + assert nested_field.node.properties[0].provenance is not None + assert nested_field.node.properties[0].provenance.path == ("nested", "inner") + + +def test_read_node_pass_standalone_list_provenance() -> None: + """read_node attaches empty root provenance and element path for standalone lists.""" + standalone_node = IcebergReader().read_node(ListType(10, StringType(), element_required=False)) + assert standalone_node.provenance is not None + assert standalone_node.provenance.path == () + assert standalone_node.items is not None + assert standalone_node.items.provenance is not None + assert standalone_node.items.provenance.path == ("element",) + + +def test_read_field_pass_standalone_map_provenance() -> None: + """read_field attaches field, key, and value provenance paths for standalone maps.""" + standalone_field = IcebergReader().read_field(NestedField(11, "col", MapType(12, StringType(), 13, IntegerType()))) + assert standalone_field.provenance is not None + assert standalone_field.provenance.path == ("col",) + assert standalone_field.annotations == {"description": None} + assert standalone_field.node.key_type is not None + assert standalone_field.node.key_type.provenance is not None + assert standalone_field.node.key_type.provenance.path == ("col", "key") + assert standalone_field.node.value_type is not None + assert standalone_field.node.value_type.provenance is not None + assert standalone_field.node.value_type.provenance.path == ("col", "value") + + +def test_read_table_pass_without_comment() -> None: + """read_table leaves annotations empty and records identifier and format version when no comment is set.""" + table_no_comment = make_table(("ns", "plain"), Schema(NestedField(1, "id", LongType(), required=True))) + table_doc = IcebergReader().read_table(table_no_comment, ("ns", "plain")) + assert table_doc.annotations == {} + assert table_doc.provenance is not None + assert table_doc.provenance.path == ("ns", "plain") + assert "format-version" in table_doc.provenance.metadata diff --git a/tests/cdm_data_loaders/converters/sources/test_pyarrow_reader.py b/tests/cdm_data_loaders/converters/sources/test_pyarrow_reader.py new file mode 100644 index 00000000..bb72ee4a --- /dev/null +++ b/tests/cdm_data_loaders/converters/sources/test_pyarrow_reader.py @@ -0,0 +1,291 @@ +"""Unit tests for cdm_data_loaders.converters.sources.pyarrow.PyArrowReader.""" + +from typing import Any, Final + +import pyarrow as pa +import pytest + +from cdm_data_loaders.converters.core.ir import NodeHints, Provenance, TypedNode +from cdm_data_loaders.converters.exceptions.sources.pyarrow import PyArrowReaderError +from cdm_data_loaders.converters.sources.pyarrow import PyArrowReader, _format_path + + +class _DummyExtType(pa.ExtensionType): + def __init__(self) -> None: + super().__init__(pa.int64(), "dummy.ext") + + def __arrow_ext_serialize__(self) -> bytes: + return b"" + + @classmethod + def __arrow_ext_deserialize__(cls, _storage_type: pa.DataType, _serialized: bytes) -> "_DummyExtType": + return cls() + + +LIST_ELEM_FIELD: Final[pa.Field] = pa.field( + "item", pa.int64(), nullable=False, metadata={"comment": "elem desc", "k": "v"} +) + + +@pytest.mark.parametrize( + ("path", "expected"), + [ + pytest.param((), "", id="empty-root"), + pytest.param(("a", 0, "b"), "a.0.b", id="nested-segments"), + ], +) +def test_format_path_segments(path: tuple[str | int, ...], expected: str) -> None: + """Verify _format_path formats empty and nested path tuples.""" + assert _format_path(path) == expected + + +@pytest.mark.parametrize( + ("bad_schema", "pattern"), + [ + pytest.param("not_a_schema", r"Expected a pyarrow\.Schema", id="non-schema"), + pytest.param(pa.schema([]), "Empty PyArrow schema with 0 fields", id="empty-schema"), + pytest.param( + pa.schema([pa.field("dup", pa.int64()), pa.field("dup", pa.string())]), + "Duplicate field name 'dup' in PyArrow schema", + id="duplicate-field", + ), + ], +) +def test_pyarrow_reader_read_invalid_input_raises(bad_schema: object, pattern: str) -> None: + """Verify PyArrowReader.read rejects invalid, empty, or duplicate-field schemas.""" + with pytest.raises(PyArrowReaderError, match=pattern): + PyArrowReader().read(bad_schema) # type: ignore[arg-type] + + +def test_pyarrow_reader_read_field_invalid_type_raises() -> None: + """Verify PyArrowReader.read_field rejects non-Field inputs.""" + with pytest.raises(PyArrowReaderError, match=r"Expected a pyarrow\.Field"): + PyArrowReader().read_field("not_a_field") # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("bad_type", "path", "pattern"), + [ + pytest.param("not_a_type", (), r"Expected a pyarrow\.DataType", id="non-datatype"), + pytest.param(pa.struct([]), ("nested",), "Empty PyArrow struct with 0 fields", id="empty-struct"), + pytest.param( + pa.struct([pa.field("x", pa.int64()), pa.field("x", pa.string())]), + ("nested",), + "Duplicate field name 'x' in PyArrow struct at 'nested'", + id="duplicate-struct-field", + ), + pytest.param(pa.null(), ("col",), "Standalone PyArrow null type", id="standalone-null"), + pytest.param(_DummyExtType(), ("col",), "Unsupported PyArrow extension type", id="extension-type"), + pytest.param(pa.duration("s"), ("col",), "duration/interval", id="duration"), + pytest.param(pa.month_day_nano_interval(), ("col",), "duration/interval", id="interval"), + pytest.param(pa.union([pa.field("a", pa.int32())], mode="sparse"), ("col",), "union type", id="union"), + pytest.param(pa.run_end_encoded(pa.int32(), pa.string()), ("col",), "run_end_encoded", id="run-end-encoded"), + pytest.param(pa.decimal128(5, -1), ("col",), "Invalid PyArrow decimal", id="decimal-negative-scale"), + pytest.param(pa.decimal128(5, 6), ("col",), "Invalid PyArrow decimal", id="decimal-scale-exceeds-precision"), + ], +) +def test_pyarrow_reader_read_node_invalid_input_raises( + bad_type: object, + path: tuple[str | int, ...], + pattern: str, +) -> None: + """Verify PyArrowReader.read_node rejects unsupported and malformed PyArrow data types.""" + with pytest.raises(PyArrowReaderError, match=pattern): + PyArrowReader().read_node(bad_type, path=path) # type: ignore[arg-type] + + +def test_pyarrow_reader_read_preserves_metadata_and_requiredness() -> None: + """Verify PyArrowReader.read preserves schema metadata, annotations, provenance, and required fields.""" + schema = pa.schema( + [ + pa.field("req_id", pa.int64(), nullable=False, metadata={"comment": "ID", "pydantic_type": "int"}), + pa.field("opt_name", pa.string(), nullable=True, metadata={"custom": "v"}), + ], + metadata={"title": "MetaTitle", "$id": "https://example.org/meta", "comment": "Doc desc", "extra": "keep"}, + ) + doc = PyArrowReader().read(schema) + + assert doc.name == "MetaTitle" + assert doc.identifier == "https://example.org/meta" + assert doc.annotations["description"] == "Doc desc" + assert doc.root.annotations["description"] == "Doc desc" + assert doc.extensions["x-pyarrow"]["metadata"] == {"extra": "keep"} + assert doc.root.required_names == ("req_id",) + assert doc.root.additional_properties == TypedNode(type="never", nullable=False) + assert doc.provenance == Provenance("pyarrow", ()) + assert doc.root.provenance == Provenance("pyarrow", ()) + + +def test_pyarrow_reader_read_overrides_name_and_id() -> None: + """Verify PyArrowReader.read uses explicit schema_name and schema_id overrides over metadata.""" + schema = pa.schema( + [pa.field("a", pa.int32())], + metadata={"title": "MetaTitle", "$id": "https://example.org/meta"}, + ) + doc = PyArrowReader().read(schema, schema_name="OverrideName", schema_id="https://example.org/override") + + assert doc.name == "OverrideName" + assert doc.identifier == "https://example.org/override" + + +def test_pyarrow_reader_read_excludes_metadata_when_disabled() -> None: + """Verify PyArrowReader omits x-pyarrow metadata when include_pyarrow_metadata is False.""" + doc = PyArrowReader(include_pyarrow_metadata=False).read(pa.schema([pa.field("a", pa.int32())])) + + assert doc.name is None + assert doc.identifier is None + assert not doc.annotations + assert not doc.extensions + + +def test_pyarrow_reader_read_field_metadata_and_provenance() -> None: + """Verify PyArrowReader.read_field preserves requiredness, description, provenance, and extensions.""" + field = PyArrowReader().read_field( + pa.field("col", pa.int32(), nullable=False, metadata={"comment": "c", "xsd_type": "xs:int"}) + ) + + assert field.required is True + assert field.annotations["description"] == "c" + assert field.provenance == Provenance("pyarrow", ("col",), {"xsd_type": "xs:int"}) + assert field.extensions["x-pyarrow"]["metadata"] == {"xsd_type": "xs:int"} + + +@pytest.mark.parametrize( + ("list_type", "expected_arrow"), + [ + pytest.param(pa.list_(LIST_ELEM_FIELD), "list", id="list"), + pytest.param(pa.large_list(LIST_ELEM_FIELD), "large_list", id="large-list"), + pytest.param(pa.list_view(LIST_ELEM_FIELD), "list_view", id="list-view"), + pytest.param(pa.large_list_view(LIST_ELEM_FIELD), "large_list_view", id="large-list-view"), + pytest.param(pa.list_(LIST_ELEM_FIELD, 3), "fixed_size_list", id="fixed-size-list"), + ], +) +@pytest.mark.parametrize( + "include_metadata", + [ + pytest.param(True, id="with-metadata"), + pytest.param(False, id="without-metadata"), + ], +) +def test_pyarrow_reader_read_node_list_variants( + list_type: pa.DataType, + expected_arrow: str, + include_metadata: bool, +) -> None: + """Verify PyArrowReader.read_node handles all PyArrow list variants with and without metadata.""" + node = PyArrowReader(include_pyarrow_metadata=include_metadata).read_node(list_type, nullable=True, path=("arr",)) + + assert node.type == "array" + assert node.nullable is True + assert isinstance(node.items, TypedNode) + assert node.items.annotations["description"] == "elem desc" + if include_metadata: + assert node.extensions["x-pyarrow"]["arrow_type"] == expected_arrow + assert node.items.extensions["x-pyarrow"]["metadata"] == {"k": "v"} + else: + assert not node.items.extensions + + +@pytest.mark.parametrize( + ("include_metadata", "ordered"), + [ + pytest.param(True, True, id="ordered-with-metadata"), + pytest.param(False, False, id="unordered-without-metadata"), + ], +) +def test_pyarrow_reader_read_node_dictionary(include_metadata: bool, ordered: bool) -> None: + """Verify PyArrowReader.read_node lowers dictionary types and preserves ordering metadata when enabled.""" + node = PyArrowReader(include_pyarrow_metadata=include_metadata).read_node( + pa.dictionary(pa.int8(), pa.string(), ordered=ordered), + nullable=True, + ) + + assert node.type == "string" + assert node.nullable is True + if include_metadata: + assert node.extensions["x-pyarrow"]["arrow_type"] == "dictionary" + assert node.extensions["x-pyarrow"]["ordered"] is True + else: + assert not node.extensions + + +def test_pyarrow_reader_read_node_map() -> None: + """Verify PyArrowReader.read_node lowers PyArrow map types with non-null keys and keys_sorted metadata.""" + node = PyArrowReader().read_node(pa.map_(pa.string(), pa.int32(), keys_sorted=True)) + + assert node.type == "map" + assert node.key_type is not None + assert node.key_type.nullable is False + assert node.extensions["x-pyarrow"]["keys_sorted"] is True + + +def test_pyarrow_reader_read_node_struct() -> None: + """Verify PyArrowReader.read_node lowers PyArrow struct types and tracks required fields.""" + node = PyArrowReader().read_node(pa.struct([pa.field("a", pa.bool_(), nullable=False)])) + + assert node.type == "object" + assert node.required_names == ("a",) + + +@pytest.mark.parametrize( + ("arrow_type", "expected_hints", "expected_arrow_meta", "absent_meta_keys"), + [ + pytest.param( + pa.decimal256(40, 4), + NodeHints(logical_type="decimal", precision=40, scale=4, bit_width=256), + {}, + (), + id="decimal256", + ), + pytest.param( + pa.binary(8), + NodeHints(logical_type="fixed", length=8), + {}, + (), + id="fixed-size-binary", + ), + pytest.param( + pa.time32("s"), + None, + {"arrow_type": "time32"}, + (), + id="time32", + ), + pytest.param( + pa.time64("ns"), + None, + {"arrow_type": "time64"}, + (), + id="time64", + ), + pytest.param( + pa.timestamp("us"), + NodeHints(logical_type="timestamp-without-tz", timezone=False), + {}, + ("tz",), + id="timestamp-without-tz", + ), + pytest.param( + pa.timestamp("us", tz="UTC"), + NodeHints(logical_type="timestamp-with-tz", timezone=True), + {"tz": "UTC"}, + (), + id="timestamp-with-tz", + ), + ], +) +def test_pyarrow_reader_read_node_scalars( + arrow_type: pa.DataType, + expected_hints: NodeHints | None, + expected_arrow_meta: dict[str, Any], + absent_meta_keys: tuple[str, ...], +) -> None: + """Verify PyArrowReader.read_node maps scalar PyArrow types to hints and x-pyarrow metadata.""" + node = PyArrowReader().read_node(arrow_type) + + if expected_hints is not None: + assert node.hints == expected_hints + for key, val in expected_arrow_meta.items(): + assert node.extensions["x-pyarrow"][key] == val + for key in absent_meta_keys: + assert key not in node.extensions["x-pyarrow"] diff --git a/tests/cdm_data_loaders/converters/sources/test_pydantic_annotations.py b/tests/cdm_data_loaders/converters/sources/test_pydantic_annotations.py new file mode 100644 index 00000000..ef0395a2 --- /dev/null +++ b/tests/cdm_data_loaders/converters/sources/test_pydantic_annotations.py @@ -0,0 +1,279 @@ +"""Unit tests for Pydantic annotation unwrapping and classification in cdm_data_loaders.converters.sources.pydantic.annotations.""" + +import re +from collections.abc import Callable +from decimal import Decimal +from enum import StrEnum +from typing import Annotated, Any, Final, Literal, NewType, TypeVar, Unpack + +import pytest +from pydantic import Base64Bytes, BaseModel, ConfigDict, Field, NaiveDatetime, RootModel, computed_field + +from cdm_data_loaders.converters.core.errors import ConversionError +from cdm_data_loaders.converters.sources.pydantic.annotations import ( + ParsedAnnotation, + allows_null, + classify_parsed_kind, + extract_model_field_specs, + is_valid_string_map_key, + is_variadic_unpack, + substitute_typevars, + unwrap_annotation, +) +from cdm_data_loaders.converters.sources.pydantic.model_synthesis import SchemaBuildContext + +type MyBox[T] = list[T] +type SimpleAlias = Annotated[int, Field(description="from alias")] +UserId = NewType("UserId", str) +SAMPLE_TYPEVAR: Final = TypeVar("SAMPLE_TYPEVAR") + + +class _Color(StrEnum): + RED = "red" + + +class _DocModel(BaseModel): + """Docstring from model.""" + + x: int + + +class _RootStr(RootModel[str]): + pass + + +class _ExtraAllowModel(BaseModel): + model_config = ConfigDict(extra="allow") + x: int + + +class _RichModel(BaseModel): + kept: int = Field(alias="keptAlias", serialization_alias="keptSer", description="kept desc") + hidden: str = Field(default="x", exclude=True) + opt: str | None = None + + @computed_field(alias="compAlias", description="comp desc") # type: ignore[prop-decorator] + @property + def total(self) -> int: + return self.kept + + +class _AliasFallbackModel(BaseModel): + a_field: int = Field(alias="aAlias") + default_val: int = 5 + default_none: int = None # type: ignore[assignment] + + @computed_field(alias="cAlias") # type: ignore[prop-decorator] + @property + def comp_opt(self) -> str | None: + return None + + +def _ctx(*, by_alias: bool = False, module_ns: dict[str, Any] | None = None) -> SchemaBuildContext: + return SchemaBuildContext( + model_counts={}, + max_recursion_depth=3, + by_alias=by_alias, + error_type=ConversionError, + error_by_code={}, + module_ns=module_ns or {}, + ) + + +@pytest.mark.parametrize( + ("annotation", "mapping", "expected"), + [ + pytest.param(SAMPLE_TYPEVAR, {SAMPLE_TYPEVAR: int}, int, id="direct-typevar"), + pytest.param(Annotated[SAMPLE_TYPEVAR, "m"], {SAMPLE_TYPEVAR: str}, Annotated[str, "m"], id="annotated"), + pytest.param(SAMPLE_TYPEVAR | None, {SAMPLE_TYPEVAR: int}, int | None, id="union"), + pytest.param(list[SAMPLE_TYPEVAR], {SAMPLE_TYPEVAR: float}, list[float], id="generic-list"), + ], +) +def test_substitute_typevars_replaces_parameters( + annotation: object, + mapping: dict[TypeVar, Any], + expected: object, +) -> None: + """Verify substitute_typevars replaces TypeVars across direct, annotated, union, and generic forms.""" + assert substitute_typevars(annotation, mapping) == expected + + +@pytest.mark.parametrize( + ("annotation", "expected_attrs"), + [ + pytest.param(MyBox[int], {"annotation": list[int]}, id="generic-type-alias"), + pytest.param(SimpleAlias, {"annotation": int, "description": "from alias"}, id="annotated-type-alias"), + pytest.param("TargetStr", {"annotation": str}, id="forward-ref"), + pytest.param(UserId, {"annotation": str}, id="new-type"), + pytest.param(_RootStr, {"annotation": str}, id="root-model"), + pytest.param( + Annotated[Decimal, Field(max_digits=10, decimal_places=3)], + {"max_digits": 10, "decimal_places": 3}, + id="decimal-constraints", + ), + pytest.param(NaiveDatetime, {"is_naive_datetime": True}, id="naive-datetime"), + pytest.param(Base64Bytes, {"is_base64_bytes": True}, id="base64-bytes"), + pytest.param(_DocModel | None, {"description": "Docstring from model."}, id="optional-model-docstring"), + ], +) +def test_unwrap_annotation_variants(annotation: object, expected_attrs: dict[str, Any]) -> None: + """Verify unwrap_annotation resolves aliases, forward refs, NewType, RootModel, and metadata.""" + parsed, _ = unwrap_annotation(annotation, _ctx(module_ns={"TargetStr": str})) + for attr, expected in expected_attrs.items(): + assert getattr(parsed, attr) == expected + + +def test_unwrap_annotation_missing_forward_ref_raises() -> None: + """Verify unwrap_annotation raises ConversionError for unresolved forward references.""" + with pytest.raises(ConversionError): + unwrap_annotation("MissingRef", _ctx(module_ns={"TargetStr": str})) + + +@pytest.mark.parametrize( + ("annotation", "expected"), + [ + pytest.param(None, True, id="none"), + pytest.param(Literal.__getitem__((None, "a")), True, id="literal-with-none"), + pytest.param(int | None, True, id="optional-int"), + pytest.param(str, False, id="non-null-str"), + ], +) +def test_allows_null_variants(annotation: object, expected: bool) -> None: + """Verify allows_null detects nullable annotations and rejects non-nullable ones.""" + assert allows_null(annotation, _ctx()) is expected + + +@pytest.mark.parametrize( + ("annotation", "expected_kind"), + [ + pytest.param(None, "null", id="null"), + pytest.param(list[int], "sequence", id="list"), + pytest.param(tuple[int, ...], "sequence", id="variadic-tuple"), + pytest.param(tuple[int, str], "tuple", id="fixed-tuple"), + pytest.param(dict[str, int], "map", id="dict-map"), + pytest.param(_DocModel, "model", id="model"), + pytest.param(int, "scalar", id="scalar"), + ], +) +def test_classify_parsed_kind_valid(annotation: object, expected_kind: str) -> None: + """Verify classify_parsed_kind classifies supported annotations into IR kinds.""" + assert classify_parsed_kind(ParsedAnnotation(annotation), _ctx()) == expected_kind + + +@pytest.mark.parametrize( + "bad_annotation", + [ + pytest.param(Any, id="any"), + pytest.param(object, id="object"), + pytest.param(TypeVar("U"), id="typevar"), + pytest.param(Callable[[], None], id="callable"), + pytest.param(re.Pattern, id="regex-pattern"), + pytest.param(list, id="untyped-list"), + pytest.param(tuple, id="untyped-tuple"), + pytest.param(dict, id="untyped-dict"), + pytest.param(BaseModel, id="bare-base-model"), + pytest.param(RootModel, id="bare-root-model"), + ], +) +def test_classify_parsed_kind_unsupported_raises(bad_annotation: object) -> None: + """Verify classify_parsed_kind raises ConversionError for unsupported annotations.""" + with pytest.raises(ConversionError): + classify_parsed_kind(ParsedAnnotation(bad_annotation), _ctx()) + + +@pytest.mark.parametrize( + ("key_type", "expected"), + [ + pytest.param(str, True, id="str"), + pytest.param(Literal["a", _Color.RED], True, id="literal-str-and-strenum"), + pytest.param(_Color | Literal["b"] | None, True, id="strenum-literal-optional-union"), + pytest.param(int, False, id="int"), + pytest.param(None, False, id="none"), + pytest.param(Annotated[None, "a"] | Annotated[None, "b"], False, id="annotated-none-union"), + pytest.param(Literal[1, "a"], False, id="mixed-literal"), + ], +) +def test_is_valid_string_map_key_variants(key_type: object, expected: bool) -> None: + """Verify is_valid_string_map_key accepts string-like keys and rejects non-string keys.""" + assert is_valid_string_map_key(key_type, _ctx()) is expected + + +@pytest.mark.parametrize( + "bad_key", + [ + pytest.param(Any, id="any"), + pytest.param(object, id="object"), + pytest.param(TypeVar("K"), id="typevar"), + ], +) +def test_is_valid_string_map_key_unsupported_raises(bad_key: object) -> None: + """Verify is_valid_string_map_key raises ConversionError for unsupported key types.""" + with pytest.raises(ConversionError): + is_valid_string_map_key(bad_key, _ctx()) + + +@pytest.mark.parametrize( + ("annotation", "expected"), + [ + pytest.param(Ellipsis, True, id="ellipsis"), + pytest.param(Unpack[tuple[int, ...]], True, id="unpack-variadic-tuple"), + pytest.param(Unpack[tuple], True, id="unpack-untyped-tuple"), + pytest.param(Unpack[tuple[int, str]], False, id="unpack-fixed-tuple"), + pytest.param(int, False, id="scalar-int"), + ], +) +def test_is_variadic_unpack_variants(annotation: object, expected: bool) -> None: + """Verify is_variadic_unpack distinguishes variadic tuple unpacks from fixed types.""" + assert is_variadic_unpack(annotation) is expected + + +def test_extract_model_field_specs_extra_allow_raises() -> None: + """Verify extract_model_field_specs rejects models with extra='allow'.""" + with pytest.raises(ConversionError): + extract_model_field_specs(_ExtraAllowModel, _ctx()) + + +def test_extract_model_field_specs_excluded_aliases_and_computed() -> None: + """Verify extract_model_field_specs omits excluded fields and resolves serialization and computed aliases.""" + specs = extract_model_field_specs(_RichModel, _ctx(by_alias=True)) + by_name = {s.output_name: s for s in specs} + + assert "hidden" not in by_name + assert by_name["keptSer"].required_non_null is True + assert by_name["keptSer"].description == "kept desc" + assert by_name["opt"].required_non_null is False + assert by_name["compAlias"].annotation is int + assert by_name["compAlias"].description == "comp desc" + + +@pytest.mark.parametrize( + ("by_alias", "expected_names", "expected_required"), + [ + pytest.param( + False, + {"a_field", "default_val", "default_none", "comp_opt"}, + {"a_field": True, "default_val": True, "default_none": False, "comp_opt": False}, + id="python-names", + ), + pytest.param( + True, + {"aAlias", "default_val", "default_none", "cAlias"}, + {"aAlias": True, "default_val": True, "default_none": False, "cAlias": False}, + id="alias-names", + ), + ], +) +def test_extract_model_field_specs_alias_mode_and_defaults( + by_alias: bool, + expected_names: set[str], + expected_required: dict[str, bool], +) -> None: + """Verify extract_model_field_specs honors by_alias and marks default=None fields nullable.""" + specs = extract_model_field_specs(_AliasFallbackModel, _ctx(by_alias=by_alias)) + by_name = {s.output_name: s for s in specs} + + assert set(by_name) == expected_names + for name, req in expected_required.items(): + assert by_name[name].required_non_null is req + first_spec = next(iter(by_name.values())) + assert first_spec.context.model_counts[_AliasFallbackModel] == 1 diff --git a/tests/cdm_data_loaders/converters/sources/test_pydantic_model_synthesis.py b/tests/cdm_data_loaders/converters/sources/test_pydantic_model_synthesis.py new file mode 100644 index 00000000..db30429e --- /dev/null +++ b/tests/cdm_data_loaders/converters/sources/test_pydantic_model_synthesis.py @@ -0,0 +1,374 @@ +"""Unit tests for cdm_data_loaders.converters.sources.pydantic.model_synthesis.""" + +from dataclasses import dataclass, field +from enum import Enum, IntEnum, StrEnum +from typing import Any, ForwardRef, NotRequired, Required, TypedDict + +import pytest +from frozendict import frozendict +from pydantic import BaseModel, PydanticSchemaGenerationError, RootModel, create_model + +from cdm_data_loaders.converters.core.errors import ConversionError +from cdm_data_loaders.converters.sources.pydantic.model_synthesis import ( + SchemaBuildContext, + clean_docstring, + coerce_structured_type_to_model, + dataclass_to_model, + ensure_concrete_model, + ensure_model_complete, + extract_class_docstring, + safe_type_hints, + typeddict_to_model, + validate_model_class, +) + + +class CustomConvError(ConversionError): + """Custom conversion error for context tests.""" + + +class SampleModel(BaseModel): + """Sample model doc.""" + + x: int + + +class SampleEnum(Enum): + """Sample enum doc.""" + + A = "a" + + +class SampleIntEnum(IntEnum): + """Sample int enum doc.""" + + ONE = 1 + + +class SampleStrEnum(StrEnum): + """Sample str enum doc.""" + + FOO = "foo" + + +class _NoDocChildModel(SampleModel): + y: str + + +class SampleTypedDict(TypedDict, total=False): + """TypedDict doc.""" + + req_field: Required[int] + opt_field: NotRequired[str] + plain_opt: float + + +class _TotalTypedDict(TypedDict): + a: int + + +@dataclass +class SampleDataclass: + """Dataclass doc. + + More details. + """ + + req: int + with_default: str = "abc" + with_factory: list[int] = field(default_factory=list) + + +@dataclass +class _AutoDocDataclass: + x: int + + +class _UnresolvedHints: + pass + + +_UnresolvedHints.__annotations__ = {"x": "MissingHintType"} + + +def _error_ctx() -> SchemaBuildContext: + return SchemaBuildContext( + error_by_code=frozendict({"E1": CustomConvError, "E4": CustomConvError, "E9": CustomConvError}) + ) + + +@pytest.mark.parametrize( + ("code", "message", "cause", "expected_exc"), + [ + pytest.param("E4", "bad generic", ValueError("boom"), CustomConvError, id="mapped-code-with-cause"), + pytest.param("E99", "other", None, ConversionError, id="unmapped-code-fallback"), + ], +) +def test_schema_build_context_raise_err( + code: str, + message: str, + cause: Exception | None, + expected_exc: type[ConversionError], +) -> None: + """Verify SchemaBuildContext.raise_err selects mapped or fallback errors and preserves cause.""" + ctx = SchemaBuildContext( + max_recursion_depth=5, + by_alias=True, + error_type=ConversionError, + error_by_code=frozendict({"E4": CustomConvError}), + ) + + with pytest.raises(expected_exc, match=message) as exc_info: + ctx.raise_err(code, message, cause=cause) + assert exc_info.value.__cause__ is cause + + +def test_schema_build_context_with_alias_and_with_model() -> None: + """Verify with_alias and with_model update active aliases, model counts, and module namespace immutably.""" + ctx = SchemaBuildContext( + max_recursion_depth=5, + by_alias=True, + error_type=ConversionError, + error_by_code=frozendict({"E4": CustomConvError}), + ) + + alias_ctx = ctx.with_alias("AliasObj") + assert alias_ctx.active_aliases == frozenset({"AliasObj"}) + assert alias_ctx.max_recursion_depth == ctx.max_recursion_depth + assert alias_ctx.by_alias is True + assert alias_ctx.error_by_code == ctx.error_by_code + + model_ctx = alias_ctx.with_model(SampleModel) + assert model_ctx.model_counts[SampleModel] == 1 + assert model_ctx.active_aliases == frozenset() + assert model_ctx.max_recursion_depth == ctx.max_recursion_depth + assert model_ctx.by_alias is True + assert model_ctx.error_by_code == ctx.error_by_code + assert "SampleModel" in model_ctx.module_ns + + model_ctx2 = model_ctx.with_model(SampleModel) + expected_count = 2 + assert model_ctx2.model_counts[SampleModel] == expected_count + + +def test_schema_build_context_with_model_missing_module_preserves_ns() -> None: + """Verify with_model keeps the existing module_ns when the model module is not in sys.modules.""" + + class MissingModModel(BaseModel): + a: int + + MissingModModel.__module__ = "nonexistent_mod_xyz_123" + custom_ns = frozendict({"sentinel": 42}) + ctx_with_ns = SchemaBuildContext(module_ns=custom_ns) + + assert ctx_with_ns.with_model(MissingModModel).module_ns == custom_ns + + +def test_validate_model_class_valid() -> None: + """Verify validate_model_class returns a concrete BaseModel subclass unchanged.""" + assert validate_model_class(SampleModel, _error_ctx()) is SampleModel + + +@pytest.mark.parametrize( + "invalid", + [ + pytest.param(123, id="non-type"), + pytest.param(dict, id="non-model-class"), + pytest.param(BaseModel, id="bare-base-model"), + pytest.param(RootModel, id="bare-root-model"), + ], +) +def test_validate_model_class_invalid_raises(invalid: object) -> None: + """Verify validate_model_class rejects non-model types and abstract Pydantic base classes.""" + with pytest.raises(CustomConvError): + validate_model_class(invalid, _error_ctx()) + + +def test_ensure_model_complete_rebuilds_incomplete_model() -> None: + """Verify ensure_model_complete rebuilds an incomplete model with resolvable forward references.""" + + class IncompleteModel(BaseModel): + ref: "SampleModel" + + IncompleteModel.__pydantic_complete__ = False + ensure_model_complete(IncompleteModel, _error_ctx()) + + assert IncompleteModel.__pydantic_complete__ is True + + +@pytest.mark.parametrize( + ("ctx", "expected_exc"), + [ + pytest.param(_error_ctx(), CustomConvError, id="with-context"), + pytest.param(None, ConversionError, id="without-context"), + ], +) +def test_ensure_model_complete_unresolved_forward_ref_raises( + ctx: SchemaBuildContext | None, + expected_exc: type[ConversionError], +) -> None: + """Verify ensure_model_complete raises on unresolved forward references.""" + broken_ref_model = create_model("BrokenRefModel", bad=(ForwardRef("DoesNotExistXYZ"), ...)) + + with pytest.raises(expected_exc, match="Unresolved forward reference"): + ensure_model_complete(broken_ref_model, ctx) + + +def test_ensure_model_complete_ignores_schema_generation_error(monkeypatch: pytest.MonkeyPatch) -> None: + """Verify ensure_model_complete ignores PydanticSchemaGenerationError during model_rebuild.""" + + class SchemaGenErrModel(BaseModel): + x: int + + def _raise_schema_gen(_cls: type[BaseModel], **_kwargs: object) -> bool: + msg = "skip" + raise PydanticSchemaGenerationError(msg) + + SchemaGenErrModel.__pydantic_complete__ = False + monkeypatch.setattr(SchemaGenErrModel, "model_rebuild", classmethod(_raise_schema_gen)) + + ensure_model_complete(SchemaGenErrModel, _error_ctx()) + + +@pytest.mark.parametrize( + ("ctx", "expected_exc"), + [ + pytest.param(_error_ctx(), CustomConvError, id="with-context"), + pytest.param(None, ConversionError, id="without-context"), + ], +) +def test_ensure_model_complete_still_incomplete_raises( + monkeypatch: pytest.MonkeyPatch, + ctx: SchemaBuildContext | None, + expected_exc: type[ConversionError], +) -> None: + """Verify ensure_model_complete raises when model_rebuild returns False.""" + + class StillIncompleteModel(BaseModel): + x: int + + def _return_false(_cls: type[BaseModel], **_kwargs: object) -> bool: + return False + + StillIncompleteModel.__pydantic_complete__ = False + monkeypatch.setattr(StillIncompleteModel, "model_rebuild", classmethod(_return_false)) + + with pytest.raises(expected_exc, match="remains incomplete"): + ensure_model_complete(StillIncompleteModel, ctx) + + +def test_ensure_concrete_model_unparameterized_generic_raises() -> None: + """Verify ensure_concrete_model rejects unparameterized generic models.""" + + class GenericBox[T](BaseModel): + val: T + + with pytest.raises(CustomConvError, match="Unparameterized generic model"): + ensure_concrete_model(GenericBox, _error_ctx()) + + +@pytest.mark.parametrize( + ("cls", "expected"), + [ + pytest.param(SampleModel, "Sample model doc.", id="model-with-doc"), + pytest.param(_NoDocChildModel, None, id="subclass-without-own-doc"), + pytest.param(_AutoDocDataclass, None, id="auto-generated-dataclass-doc"), + pytest.param(SampleDataclass, "Dataclass doc.\n\nMore details.", id="multiline-dataclass-doc"), + ], +) +def test_clean_docstring_variants(cls: type[Any], expected: str | None) -> None: + """Verify clean_docstring strips indentation and ignores inherited or auto-generated docstrings.""" + assert clean_docstring(cls) == expected + + +@pytest.mark.parametrize( + ("target", "expected"), + [ + pytest.param("not a type", None, id="non-type"), + pytest.param(dict, None, id="builtin-dict"), + pytest.param(BaseModel, None, id="base-model"), + pytest.param(RootModel, None, id="root-model"), + pytest.param(Enum, None, id="enum-base"), + pytest.param(IntEnum, None, id="int-enum-base"), + pytest.param(StrEnum, None, id="str-enum-base"), + pytest.param(SampleModel, "Sample model doc.", id="sample-model"), + pytest.param(SampleEnum, "Sample enum doc.", id="sample-enum"), + pytest.param(SampleIntEnum, "Sample int enum doc.", id="sample-int-enum"), + pytest.param(SampleStrEnum, "Sample str enum doc.", id="sample-str-enum"), + ], +) +def test_extract_class_docstring_variants(target: object, expected: str | None) -> None: + """Verify extract_class_docstring returns user docstrings and skips non-types and base classes.""" + assert extract_class_docstring(target) == expected + + +@pytest.mark.parametrize( + ("cls", "expected"), + [ + pytest.param(_UnresolvedHints, {"x": "MissingHintType"}, id="unresolved-annotations-fallback"), + pytest.param(_TotalTypedDict, {"a": int}, id="resolved-typeddict"), + ], +) +def test_safe_type_hints_variants(cls: type[Any], expected: dict[str, Any]) -> None: + """Verify safe_type_hints resolves valid annotations and falls back to raw annotations on failure.""" + assert safe_type_hints(cls, {}) == expected + + +@pytest.mark.parametrize( + ("td_cls", "expected_doc", "expected_required"), + [ + pytest.param( + SampleTypedDict, + "TypedDict doc.", + {"req_field": True, "opt_field": False, "plain_opt": False}, + id="partial-typeddict-with-doc", + ), + pytest.param( + _TotalTypedDict, + None, + {"a": True}, + id="total-typeddict-without-doc", + ), + ], +) +def test_typeddict_to_model_variants( + td_cls: type[Any], + expected_doc: str | None, + expected_required: dict[str, bool], +) -> None: + """Verify typeddict_to_model preserves docstrings and per-field requiredness.""" + model = typeddict_to_model(td_cls, SchemaBuildContext()) + + assert model.__doc__ == expected_doc + assert {name: f.is_required() for name, f in model.model_fields.items()} == expected_required + + +def test_dataclass_to_model_preserves_doc_and_defaults() -> None: + """Verify dataclass_to_model preserves docstrings, required fields, defaults, and default factories.""" + dc_model = dataclass_to_model(SampleDataclass, SchemaBuildContext()) + + assert dc_model.__doc__ == "Dataclass doc.\n\nMore details." + assert dc_model.model_fields["req"].is_required() is True + assert dc_model.model_fields["with_default"].default == "abc" + assert dc_model.model_fields["with_factory"].default_factory is list + + +@pytest.mark.parametrize( + ("source_type", "expect_synthesized_model"), + [ + pytest.param(int | str, False, id="union-passthrough"), + pytest.param(SampleModel, False, id="model-passthrough"), + pytest.param(int, False, id="scalar-passthrough"), + pytest.param(SampleTypedDict, True, id="typeddict-synthesized"), + pytest.param(SampleDataclass, True, id="dataclass-synthesized"), + ], +) +def test_coerce_structured_type_to_model_variants(source_type: object, expect_synthesized_model: bool) -> None: + """Verify coerce_structured_type_to_model synthesizes models for TypedDict and dataclass only.""" + result = coerce_structured_type_to_model(source_type, SchemaBuildContext()) + + if expect_synthesized_model: + assert isinstance(result, type) + assert issubclass(result, BaseModel) + else: + assert result == source_type diff --git a/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_unit.py b/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_unit.py new file mode 100644 index 00000000..39683916 --- /dev/null +++ b/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_unit.py @@ -0,0 +1,132 @@ +"""Unit tests for cdm_data_loaders.converters.sources.pydantic.reader.PydanticReader.""" + +from typing import Annotated, Any + +import pytest +from frozendict import frozendict +from pydantic import BaseModel, Field, RootModel, ValidationError + +from cdm_data_loaders.converters.core.errors import ConversionError +from cdm_data_loaders.converters.core.ir import Provenance +from cdm_data_loaders.converters.sources.pydantic.reader import PydanticReader + + +class _ReaderConvError(ConversionError): + """Custom error for PydanticReader error_by_code tests.""" + + +class _ModelA(BaseModel): + a_field: int = Field(alias="aAlias") + + +class _ModelB(BaseModel): + b_field: str + + +class _RootSingle(RootModel[_ModelA]): + """Root single doc.""" + + +class _RootUnion(RootModel[_ModelA | _ModelB | None]): + pass + + +def _reader() -> PydanticReader: + return PydanticReader(by_alias=True, error_by_code=frozendict({"E2": _ReaderConvError, "E7": _ReaderConvError})) + + +def test_pydantic_reader_invalid_by_alias_raises() -> None: + """Verify PydanticReader rejects non-boolean by_alias configuration values.""" + with pytest.raises(ValidationError): + PydanticReader(by_alias="yes") # type: ignore[arg-type] + + +@pytest.mark.parametrize( + "bad_root", + [ + pytest.param(RootModel[int], id="scalar-root"), + pytest.param(RootModel[int | None], id="optional-scalar-root"), + pytest.param(RootModel[None], id="none-root"), + ], +) +def test_pydantic_reader_read_invalid_root_model_raises(bad_root: type[RootModel[Any]]) -> None: + """Verify PydanticReader.read rejects RootModel types that do not wrap model objects.""" + with pytest.raises(_ReaderConvError, match="RootModel"): + _reader().read(bad_root) + + +def test_pydantic_reader_read_root_model_single() -> None: + """Verify PydanticReader.read lowers a single-model RootModel with aliases, docstrings, and provenance.""" + doc = _reader().read(_RootSingle) + + assert doc.name == "_RootSingle" + assert doc.identifier == f"urn:pydantic:{_RootSingle.__module__}._RootSingle" + assert doc.annotations == {"description": "Root single doc."} + assert doc.root.annotations == {"description": "Root single doc."} + assert doc.root.nullable is False + assert doc.root.properties[0].name == "aAlias" + assert doc.provenance == Provenance("pydantic", ("_RootSingle",)) + + +def test_pydantic_reader_read_root_model_union() -> None: + """Verify PydanticReader.read merges fields across a union RootModel.""" + doc = _reader().read(_RootUnion) + + assert doc.annotations == {} + assert doc.root.annotations == {} + assert {f.name for f in doc.root.properties} == {"aAlias", "b_field"} + + +def test_pydantic_reader_read_local_model_strips_locals_from_identifier() -> None: + """Verify PydanticReader.read omits .. segments from locally defined model identifiers.""" + + class _LocalModel(BaseModel): + x: int + + doc = _reader().read(_LocalModel) + + assert doc.identifier is not None + assert ".." not in doc.identifier + + +@pytest.mark.parametrize( + ("annotation", "expected_type", "expected_nullable", "expected_provenance"), + [ + pytest.param( + int, + "integer", + False, + Provenance("pydantic", (), {"pydantic_type": "integer"}), + id="int", + ), + pytest.param( + int | str | None, + "string", + True, + None, + id="optional-scalar-union", + ), + ], +) +def test_pydantic_reader_read_scalar_annotation_valid( + annotation: object, + expected_type: str, + expected_nullable: bool, + expected_provenance: Provenance | None, +) -> None: + """Verify read_scalar_annotation resolves scalar types, unions, and nullability.""" + reader = PydanticReader(error_by_code=frozendict({"E7": _ReaderConvError})) + node = reader.read_scalar_annotation(annotation) + + assert node.type == expected_type + assert node.nullable is expected_nullable + if expected_provenance is not None: + assert node.provenance == expected_provenance + + +def test_pydantic_reader_read_scalar_annotation_none_only_raises() -> None: + """Verify read_scalar_annotation rejects unions containing only NoneType.""" + reader = PydanticReader(error_by_code=frozendict({"E7": _ReaderConvError})) + + with pytest.raises(_ReaderConvError, match="only NoneType"): + reader.read_scalar_annotation(Annotated[None, "a"] | Annotated[None, "b"]) diff --git a/tests/cdm_data_loaders/converters/sources/test_pydantic_unions.py b/tests/cdm_data_loaders/converters/sources/test_pydantic_unions.py new file mode 100644 index 00000000..e94df891 --- /dev/null +++ b/tests/cdm_data_loaders/converters/sources/test_pydantic_unions.py @@ -0,0 +1,281 @@ +"""Unit tests for Pydantic union and container IR builders in cdm_data_loaders.converters.sources.pydantic.unions.""" + +from typing import Any + +import pytest +from pydantic import BaseModel, Field + +from cdm_data_loaders.converters.core.errors import ConversionError +from cdm_data_loaders.converters.core.ir import NodeHints, Provenance, TypedNode +from cdm_data_loaders.converters.sources.pydantic.annotations import unwrap_annotation +from cdm_data_loaders.converters.sources.pydantic.model_synthesis import SchemaBuildContext +from cdm_data_loaders.converters.sources.pydantic.unions import ( + build_field_ir, + build_merged_models_struct, + build_multi_branch_node, + build_parsed_node, +) + + +class _ModelA(BaseModel): + shared: int = Field(description="from A") + only_a: str + + +class _ModelB(BaseModel): + shared: float + only_b: bool + + +class _EmptyModel(BaseModel): + pass + + +class _ExplicitValueField(BaseModel): + val: str = Field(alias="_value") + + +class _LeafA(BaseModel): + a: int + + +class _LeafB(BaseModel): + b: str + + +def _ctx( + *, + max_recursion_depth: int = 3, + model_counts: dict[type[BaseModel], int] | None = None, + by_alias: bool = False, +) -> SchemaBuildContext: + return SchemaBuildContext( + model_counts=model_counts or {}, + max_recursion_depth=max_recursion_depth, + by_alias=by_alias, + error_type=ConversionError, + error_by_code={}, + ) + + +def test_build_field_ir_sequence() -> None: + """Verify build_field_ir lowers sequence annotations with element provenance.""" + seq_field = build_field_ir("tags", list[str], _ctx(), ("tags",), required_in_parent=True) + + assert seq_field.node.type == "array" + assert seq_field.node.nullable is False + assert seq_field.node.provenance == Provenance("pydantic", ("tags",)) + assert seq_field.node.items is not None + assert seq_field.node.items.provenance == Provenance("pydantic", ("tags", "element"), {"pydantic_type": "string"}) + + +def test_build_field_ir_tuple() -> None: + """Verify build_field_ir lowers fixed tuples to closed structs with indexed fields.""" + tup_field = build_field_ir("pair", tuple[str, int], _ctx(), ("pair",), required_in_parent=True) + + assert tup_field.node.type == "object" + assert tup_field.node.required_names == ("item_0", "item_1") + assert tup_field.node.additional_properties == TypedNode(type="never", nullable=False) + assert tup_field.node.provenance == Provenance("pydantic", ("pair",)) + assert tup_field.node.properties[0].provenance == Provenance( + "pydantic", ("pair", "item_0"), {"pydantic_type": "string"} + ) + + +def test_build_field_ir_map() -> None: + """Verify build_field_ir lowers string-keyed dict annotations to map nodes with key and value provenance.""" + map_field = build_field_ir("counts", dict[str, int], _ctx(), ("counts",), required_in_parent=False) + + assert map_field.node.type == "map" + assert map_field.node.nullable is True + assert map_field.node.provenance == Provenance("pydantic", ("counts",)) + assert map_field.node.key_type == TypedNode( + type="string", + nullable=False, + hints=NodeHints(logical_type="string"), + provenance=Provenance("pydantic", ("counts", "key")), + ) + assert map_field.node.value_type is not None + assert map_field.node.value_type.provenance == Provenance( + "pydantic", ("counts", "value"), {"pydantic_type": "integer"} + ) + + +@pytest.mark.parametrize( + "bad_ann", + [ + pytest.param(list, id="untyped-list"), + pytest.param(tuple[()], id="empty-tuple"), + pytest.param(dict, id="untyped-dict"), + pytest.param(dict[int, str], id="non-str-dict-key"), + pytest.param(None, id="none"), + ], +) +def test_build_field_ir_invalid_container_raises(bad_ann: object) -> None: + """Verify build_field_ir raises ConversionError for invalid container annotations.""" + with pytest.raises(ConversionError): + build_field_ir("bad", bad_ann, _ctx(), ("bad",), required_in_parent=True) + + +def test_build_merged_models_struct_merges_two_models() -> None: + """Verify build_merged_models_struct intersects required fields and widens shared field types.""" + merged = build_merged_models_struct([_ModelA, _ModelB], _ctx(), ("u",), nullable=False) + by_name = {f.name: f for f in merged.properties} + + assert merged.required_names == ("shared",) + assert merged.additional_properties == TypedNode(type="never", nullable=False) + assert merged.provenance == Provenance("pydantic", ("u",), {}) + assert by_name["shared"].annotations == {"description": "from A"} + assert by_name["shared"].provenance == Provenance( + "pydantic", ("u", "shared"), {"pydantic_type": "integer | number"} + ) + assert by_name["only_a"].required is False + + +def test_build_merged_models_struct_with_scalar_branches() -> None: + """Verify build_merged_models_struct synthesizes a nullable _value field for scalar union branches.""" + scalar_branch, s_ctx = unwrap_annotation(str, _ctx()) + mixed = build_merged_models_struct([_ModelA], s_ctx, ("m",), nullable=True, scalar_branches=[scalar_branch]) + by_name = {f.name: f for f in mixed.properties} + + assert mixed.required_names == () + assert by_name["_value"].required is False + assert by_name["_value"].node.nullable is True + assert by_name["_value"].provenance == Provenance("pydantic", ("m", "_value"), {"pydantic_type": "string"}) + + +def test_build_merged_models_struct_inline_struct_on_recursion() -> None: + """Verify build_merged_models_struct marks recursive model expansions with inline_struct provenance.""" + inline_ctx = _ctx(model_counts={_ModelA: 1}, max_recursion_depth=2) + inlined = build_merged_models_struct([_ModelA], inline_ctx, ("rec",), nullable=False) + + assert inlined.provenance == Provenance( + "pydantic", ("rec",), {"component_id": id(_ModelA), "type_name": "_ModelA", "inline_struct": True} + ) + + +def test_build_parsed_node_recursion_cutoff() -> None: + """Verify build_parsed_node truncates recursive models to a string node at max_recursion_depth.""" + cutoff_ctx = _ctx(model_counts={_ModelA: 1}, max_recursion_depth=1) + parsed_a, p_ctx = unwrap_annotation(_ModelA, cutoff_ctx) + cutoff_node = build_parsed_node(parsed_a, p_ctx, ("cut",), nullable=True) + + assert cutoff_node == TypedNode( + type="string", + nullable=True, + hints=NodeHints(logical_type="string"), + provenance=Provenance("pydantic", ("cut",)), + ) + + +@pytest.mark.parametrize( + ("models", "by_alias", "with_scalar"), + [ + pytest.param([_EmptyModel], False, False, id="empty-model"), + pytest.param([_ExplicitValueField], True, True, id="value-alias-collision"), + ], +) +def test_build_merged_models_struct_invalid_raises( + models: list[type[BaseModel]], + by_alias: bool, + with_scalar: bool, +) -> None: + """Verify build_merged_models_struct rejects empty models and _value alias collisions.""" + ctx = _ctx(by_alias=by_alias) + scalar_branches = [unwrap_annotation(str, ctx)[0]] if with_scalar else None + + with pytest.raises(ConversionError): + build_merged_models_struct(models, ctx, ("err",), nullable=False, scalar_branches=scalar_branches) + + +def test_build_multi_branch_node_matching_containers() -> None: + """Verify build_multi_branch_node unifies compatible sequence container branches.""" + ctx = _ctx() + branches = [unwrap_annotation(list[str], ctx), unwrap_annotation(tuple[str, ...], ctx)] + + unified = build_multi_branch_node(branches, ("c",), nullable=False) + + assert unified.type == "array" + + +@pytest.mark.parametrize( + ("annotations", "path", "nullable", "expected_type", "expected_provenance"), + [ + pytest.param( + (int, None), + ("opt_i",), + True, + "integer", + None, + id="single-non-null-with-none", + ), + pytest.param( + (int, float), + ("num",), + False, + "number", + Provenance("pydantic", ("num",), {"pydantic_type": "integer | number"}), + id="pure-scalar-union", + ), + ], +) +def test_build_multi_branch_node_scalar_variants( + annotations: tuple[Any, ...], + path: tuple[str, ...], + nullable: bool, + expected_type: str, + expected_provenance: Provenance | None, +) -> None: + """Verify build_multi_branch_node lowers optional single-scalar and multi-scalar unions.""" + ctx = _ctx() + branches = [unwrap_annotation(ann, ctx) for ann in annotations] + + node = build_multi_branch_node(branches, path, nullable=nullable) + + assert node.type == expected_type + assert node.nullable is nullable + if expected_provenance is not None: + assert node.provenance == expected_provenance + + +def test_build_multi_branch_node_model_and_scalar() -> None: + """Verify build_multi_branch_node merges model and scalar branches into an object with _value.""" + ctx = _ctx() + branches = [unwrap_annotation(_LeafA, ctx), unwrap_annotation(str, ctx)] + + node = build_multi_branch_node(branches, ("mix",), nullable=True) + + assert node.type == "object" + assert node.nullable is True + assert {f.name for f in node.properties} == {"a", "_value"} + + +def test_build_multi_branch_node_recursion_cutoff() -> None: + """Verify build_multi_branch_node falls back to string when all model branches hit recursion cutoff.""" + rec_ctx = _ctx(model_counts={_LeafA: 1, _LeafB: 1}, max_recursion_depth=1) + branches = [unwrap_annotation(_LeafA, rec_ctx), unwrap_annotation(_LeafB, rec_ctx)] + + node = build_multi_branch_node(branches, ("rec_u",), nullable=True) + + assert node == TypedNode( + type="string", + nullable=True, + hints=NodeHints(logical_type="string"), + provenance=Provenance("pydantic", ("rec_u",)), + ) + + +@pytest.mark.parametrize( + ("annotations", "nullable"), + [ + pytest.param((list[str], dict[str, int]), False, id="mismatched-containers"), + pytest.param((None,), True, id="null-only-union"), + ], +) +def test_build_multi_branch_node_invalid_raises(annotations: tuple[Any, ...], nullable: bool) -> None: + """Verify build_multi_branch_node rejects mismatched container branches and null-only unions.""" + ctx = _ctx() + branches = [unwrap_annotation(ann, ctx) for ann in annotations] + + with pytest.raises(ConversionError): + build_multi_branch_node(branches, ("bad",), nullable=nullable) diff --git a/tests/cdm_data_loaders/converters/sources/test_strict_combinators.py b/tests/cdm_data_loaders/converters/sources/test_strict_combinators.py new file mode 100644 index 00000000..b943c52a --- /dev/null +++ b/tests/cdm_data_loaders/converters/sources/test_strict_combinators.py @@ -0,0 +1,187 @@ +"""Unit tests for strict JSON Schema combinator merging and node classification.""" + +from typing import Any, Final + +import pytest + +from cdm_data_loaders.converters.core.errors import ConversionError +from cdm_data_loaders.converters.core.extensions import DEFAULT_EXTENSIONS +from cdm_data_loaders.converters.sources.json_schema.strict_combinators import ( + classify_node_kind, + classify_resolved_node_kind, + extract_choice_branches_into_object, + extract_object_parts_with_refs, + merge_property_definitions, + merge_ref_target_and_siblings, +) +from cdm_data_loaders.converters.sources.json_schema.strict_refs import StrictBuildContext + +SINGLE_PROP_DEF: Final[dict[str, Any]] = {"type": "string"} +IDENTICAL_PROP_A: Final[dict[str, Any]] = {"type": "integer", "minimum": 1} +IDENTICAL_PROP_B: Final[dict[str, Any]] = {"minimum": 1, "type": "integer"} + + +def _ctx(defs: dict[str, Any] | None = None) -> StrictBuildContext: + return StrictBuildContext( + defs=defs or {}, + ref_counts={}, + max_recursion_depth=3, + extension_registry=DEFAULT_EXTENSIONS, + error_type=ConversionError, + error_by_code={}, + ) + + +@pytest.mark.parametrize( + ("definitions", "expected", "expect_same_identity"), + [ + pytest.param([SINGLE_PROP_DEF], SINGLE_PROP_DEF, True, id="single-definition"), + pytest.param([IDENTICAL_PROP_A, IDENTICAL_PROP_B], IDENTICAL_PROP_A, True, id="identical-reordered-keys"), + pytest.param( + [{"type": "integer"}, {"type": "string", "description": "str branch"}], + { + "anyOf": [{"type": "integer"}, {"type": "string", "description": "str branch"}], + "description": "str branch", + }, + False, + id="differing-definitions-wrapped-in-anyof", + ), + ], +) +def test_merge_property_definitions_variants( + definitions: list[dict[str, Any]], + expected: dict[str, Any], + expect_same_identity: bool, +) -> None: + """Verify merge_property_definitions deduplicates identical schemas and wraps differing ones in anyOf.""" + result = merge_property_definitions(definitions) + + if expect_same_identity: + assert result is expected + else: + assert result == expected + + +def test_merge_ref_target_and_siblings_combines_fields() -> None: + """Verify merge_ref_target_and_siblings strips $id/$ref and merges properties, required, and allOf.""" + target = { + "$id": "urn:target", + "type": "object", + "properties": {"a": {"type": "integer"}, "b": {"type": "string"}}, + "required": ["a"], + "allOf": [{"properties": {"c": {"type": "boolean"}}}], + } + node = { + "$ref": "#/$defs/T", + "properties": {"a": {"type": "number"}, "d": {"type": "string"}}, + "required": ["a", "d"], + "allOf": [{"properties": {"e": {"type": "integer"}}}], + "description": "merged", + } + + merged = merge_ref_target_and_siblings(target, node) + + assert "$id" not in merged + assert "$ref" not in merged + assert merged["required"] == ["a", "d"] + assert merged["description"] == "merged" + expected_all_of_len = 2 + assert len(merged["allOf"]) == expected_all_of_len + assert merged["properties"]["a"] == {"anyOf": [{"type": "integer"}, {"type": "number"}]} + assert merged["properties"]["d"] == {"type": "string"} + + +@pytest.mark.parametrize( + ("schema", "expected_kind"), + [ + pytest.param({"anyOf": [{"type": "string"}]}, "union", id="anyof-union"), + pytest.param({"allOf": [{"properties": {"x": {"type": "int"}}}]}, "object", id="allof-object"), + pytest.param({"type": ["integer", "number"]}, "scalar", id="multi-scalar-type"), + pytest.param( + {"type": "object", "additionalProperties": {"type": "string"}}, + "map", + id="additional-properties-map", + ), + pytest.param( + {"type": "object", "properties": {"a": {"type": "string"}}, "additionalProperties": {"type": "string"}}, + "object", + id="properties-and-additional-properties-object", + ), + pytest.param({"prefixItems": [{"type": "string"}]}, "array", id="prefix-items-array"), + ], +) +def test_classify_resolved_node_kind_valid(schema: dict[str, Any], expected_kind: str) -> None: + """Verify classify_resolved_node_kind classifies union, object, scalar, map, and array schemas.""" + assert classify_resolved_node_kind(schema, error_type=ConversionError, error_by_code={}) == expected_kind + + +def test_classify_resolved_node_kind_mixed_object_and_scalar_raises() -> None: + """Verify classify_resolved_node_kind rejects mixed object and scalar type arrays.""" + with pytest.raises(ConversionError): + classify_resolved_node_kind({"type": ["object", "string"]}, error_type=ConversionError, error_by_code={}) + + +def test_classify_node_kind_ref_cycle_raises() -> None: + """Verify classify_node_kind raises ConversionError on cyclic $ref definitions.""" + defs = {"A": {"$ref": "#/$defs/B"}, "B": {"$ref": "#/$defs/A"}} + + with pytest.raises(ConversionError): + classify_node_kind( + {"$ref": "#/$defs/A"}, + defs, + extension_registry=DEFAULT_EXTENSIONS, + error_type=ConversionError, + error_by_code={}, + ) + + +def test_extract_object_parts_with_refs_merges_ref_and_conditionals() -> None: + """Verify extract_object_parts_with_refs merges $ref targets, dependentSchemas, and then/else branches.""" + defs = { + "Base": { + "type": "object", + "required": ["id"], + "properties": {"id": {"type": "string"}}, + } + } + + props, req, scalars = extract_object_parts_with_refs( + { + "$ref": "#/$defs/Base", + "required": ["extra"], + "properties": {"extra": {"type": "integer"}}, + "dependentSchemas": {"extra": {"properties": {"dep": {"type": "boolean"}}}}, + "then": {"properties": {"then_prop": {"type": "string"}}}, + "else": {"properties": {"else_prop": {"type": "string"}}}, + }, + _ctx(defs), + ) + + assert req == {"id", "extra"} + assert scalars == [] + assert set(props) == {"id", "extra", "dep", "then_prop", "else_prop"} + + +def test_extract_object_parts_with_refs_non_object_allof_raises() -> None: + """Verify extract_object_parts_with_refs rejects non-object allOf branches.""" + with pytest.raises(ConversionError): + extract_object_parts_with_refs({"allOf": [{"type": "array", "items": {"type": "string"}}]}, _ctx()) + + +@pytest.mark.parametrize( + ("branches", "keyword"), + [ + pytest.param([{"type": "null"}], "anyOf", id="null-only-anyof"), + pytest.param([{"type": "array", "items": {"type": "string"}}], "oneOf", id="array-branch-oneof"), + ], +) +def test_extract_choice_branches_into_object_invalid_raises( + branches: list[dict[str, Any]], + keyword: str, +) -> None: + """Verify extract_choice_branches_into_object rejects null-only and array choice branches.""" + raw_props: dict[str, Any] = {} + scalar_list: list[dict[str, Any]] = [] + + with pytest.raises(ConversionError): + extract_choice_branches_into_object(branches, keyword, _ctx(), raw_props, scalar_list) diff --git a/tests/cdm_data_loaders/converters/sources/test_strict_json_schema.py b/tests/cdm_data_loaders/converters/sources/test_strict_json_schema.py new file mode 100644 index 00000000..dd437622 --- /dev/null +++ b/tests/cdm_data_loaders/converters/sources/test_strict_json_schema.py @@ -0,0 +1,240 @@ +"""Unit tests for StrictJsonSchemaReader IR construction, provenance, annotations, and error handling.""" + +from typing import Any, Final + +import pytest + +from cdm_data_loaders.converters.core.errors import ConversionError +from cdm_data_loaders.converters.core.ir import Field, NodeHints, Provenance, TypedNode +from cdm_data_loaders.converters.sources.json_schema.strict_reader import StrictJsonSchemaReader + +DIALECT: Final[str] = "https://json-schema.org/draft/2020-12/schema" + + +def test_strict_json_schema_reader_read_node_scalar() -> None: + """Verify StrictJsonSchemaReader.read_node lowers a scalar fragment with constraints and provenance.""" + scalar = StrictJsonSchemaReader().read_node({"type": "integer", "minimum": 1, "description": "pos int"}) + + assert scalar.type == "integer" + assert scalar.nullable is False + assert scalar.annotations == {"description": "pos int"} + assert scalar.constraints["minimum"] == 1 + assert scalar.provenance == Provenance("json-schema", (), {"jsonschema_type": "integer"}) + + +def test_strict_json_schema_reader_read_node_array() -> None: + """Verify StrictJsonSchemaReader.read_node lowers an array fragment with item provenance.""" + arr = StrictJsonSchemaReader().read_node({"type": "array", "items": {"type": "string"}, "description": "tags"}) + + assert arr.type == "array" + assert arr.hints == NodeHints(logical_type="list") + assert arr.annotations == {"description": "tags"} + assert arr.provenance == Provenance("json-schema", ()) + assert arr.items is not None + assert arr.items.provenance == Provenance("json-schema", ("items",), {"jsonschema_type": "string"}) + + +def test_strict_json_schema_reader_read_node_tuple() -> None: + """Verify StrictJsonSchemaReader.read_node lowers a closed prefixItems tuple to a struct.""" + tup = StrictJsonSchemaReader().read_node( + { + "type": "array", + "prefixItems": [{"type": "string"}, {"type": "integer"}], + "minItems": 1, + "items": False, + "description": "pair", + } + ) + + assert tup.type == "object" + assert tup.hints == NodeHints(logical_type="struct") + assert tup.required_names == ("item_0",) + assert tup.additional_properties == TypedNode(type="never", nullable=False) + assert tup.annotations == {"description": "pair"} + assert tup.provenance == Provenance("json-schema", ()) + assert tup.properties[0] == Field( + "item_0", + TypedNode( + type="string", + declared_type="string", + nullable=False, + hints=NodeHints(logical_type="string"), + provenance=Provenance("json-schema", ("prefixItems", 0), {"jsonschema_type": "string"}), + ), + required=True, + provenance=Provenance("json-schema", ("prefixItems", 0), {"jsonschema_type": "string"}), + ) + + +def test_strict_json_schema_reader_read_node_map() -> None: + """Verify StrictJsonSchemaReader.read_node lowers an object map fragment with key and value provenance.""" + mp = StrictJsonSchemaReader().read_node( + { + "type": "object", + "additionalProperties": {"type": "integer"}, + "patternProperties": {"^x_": {"type": "integer"}}, + "description": "counts", + } + ) + + assert mp.type == "map" + assert mp.hints == NodeHints(logical_type="map") + assert mp.annotations == {"description": "counts"} + assert mp.provenance == Provenance("json-schema", ()) + assert mp.key_type == TypedNode( + type="string", + nullable=False, + hints=NodeHints(logical_type="string"), + provenance=Provenance("json-schema", (), {"jsonschema_type": "string"}), + ) + assert mp.value_type is not None + assert mp.value_type.provenance == Provenance( + "json-schema", ("additionalProperties",), {"jsonschema_type": "integer"} + ) + + +def test_strict_json_schema_reader_read_refs_unions_and_cutoff() -> None: + """Verify StrictJsonSchemaReader.read lowers $defs refs, sibling constraints, unions, and recursion cutoffs.""" + expected_min = 5 + schema: dict[str, Any] = { + "$schema": DIALECT, + "$id": "urn:test:strict", + "description": "root doc", + "$defs": { + "PosInt": {"type": "integer", "description": "from def"}, + "Node": { + "type": "object", + "required": ["label"], + "properties": { + "label": {"type": "string"}, + "next": {"$ref": "#/$defs/Node", "description": "child node"}, + }, + }, + }, + "type": "object", + "required": ["n", "u_scalar", "u_mixed"], + "properties": { + "n": {"$ref": "#/$defs/PosInt", "minimum": expected_min}, + "u_scalar": {"anyOf": [{"type": "integer"}, {"type": "number"}], "description": "num"}, + "u_mixed": { + "oneOf": [ + {"type": "string"}, + {"type": "object", "required": ["k"], "properties": {"k": {"type": "string"}}}, + ] + }, + "node": {"$ref": "#/$defs/Node"}, + }, + } + doc = StrictJsonSchemaReader(max_recursion_depth=1).read(schema) + assert doc.dialect == DIALECT + assert doc.identifier == "urn:test:strict" + assert doc.annotations == {"description": "root doc"} + assert doc.provenance == Provenance("json-schema") + + by_name = {f.name: f for f in doc.root.properties} + assert by_name["n"].node.constraints["minimum"] == expected_min + assert by_name["n"].node.annotations == {"description": "from def"} + + assert by_name["u_scalar"].node.type == "number" + assert by_name["u_scalar"].node.annotations == {"description": "num"} + assert by_name["u_scalar"].node.provenance == Provenance( + "json-schema", ("properties", "u_scalar"), {"jsonschema_type": "integer | number"} + ) + + mixed_fields = {f.name: f for f in by_name["u_mixed"].node.properties} + assert mixed_fields["k"].required is False + assert mixed_fields["_value"].required is False + assert mixed_fields["_value"].provenance == Provenance( + "json-schema", ("properties", "u_mixed", "_value"), {"jsonschema_type": "string"} + ) + + node_props = {f.name: f for f in by_name["node"].node.properties} + assert node_props["next"].node.type == "string" + assert node_props["next"].node.annotations == {"description": "child node"} + assert node_props["next"].node.provenance == Provenance("json-schema", ("properties", "node", "properties", "next")) + + +@pytest.mark.parametrize( + "fragment", + [ + pytest.param( + {"type": "array", "prefixItems": [{"type": "string"}], "items": {"type": "string"}}, id="open-tuple" + ), + pytest.param({"type": "array", "prefixItems": [], "items": False}, id="empty-prefixItems"), + pytest.param({"anyOf": [{"type": "null"}]}, id="null-only-anyOf"), + pytest.param( + {"anyOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}]}, + id="incompatible-anyOf-array-scalar", + ), + pytest.param({"type": "object", "properties": {}}, id="empty-object"), + pytest.param( + { + "oneOf": [ + {"type": "string"}, + {"type": "object", "properties": {"_value": {"type": "string"}}}, + ] + }, + id="value-collision-in-mixed-union", + ), + ], +) +def test_strict_json_schema_reader_read_node_unsupported_raises(fragment: dict[str, Any]) -> None: + """Verify StrictJsonSchemaReader.read_node raises ConversionError on unsupported strict constructs.""" + with pytest.raises(ConversionError): + StrictJsonSchemaReader().read_node(fragment) + + +def test_strict_json_schema_reader_read_inline_struct_siblings_and_object_union() -> None: + """Verify StrictJsonSchemaReader.read handles inline_struct, object siblings, and multi-object unions.""" + expected_max = 10 + expected_min = 2 + schema: dict[str, Any] = { + "$schema": DIALECT, + "$defs": { + "BoundedInt": {"type": "integer", "maximum": expected_max}, + "Node": { + "type": "object", + "required": ["label"], + "properties": { + "label": {"type": "string"}, + "next": {"$ref": "#/$defs/Node"}, + }, + }, + "PartA": {"type": "object", "required": ["a"], "properties": {"a": {"type": "integer"}}}, + "PartB": {"type": "object", "required": ["b"], "properties": {"b": {"type": "string"}}}, + }, + "type": "object", + "properties": { + "bounded": {"$ref": "#/$defs/BoundedInt", "minimum": expected_min}, + "ext_node": {"$ref": "#/$defs/Node", "properties": {"extra": {"type": "boolean"}}}, + "node_d2": {"$ref": "#/$defs/Node"}, + "opt_str": {"anyOf": [{"type": "string"}, {"type": "null"}]}, + "obj_union": {"oneOf": [{"$ref": "#/$defs/PartA"}, {"$ref": "#/$defs/PartB"}]}, + }, + } + doc = StrictJsonSchemaReader(max_recursion_depth=2).read(schema) + by_name = {f.name: f for f in doc.root.properties} + + assert by_name["bounded"].node.constraints == {"minimum": expected_min, "maximum": expected_max} + + ext_props = {f.name for f in by_name["ext_node"].node.properties} + assert ext_props == {"label", "next", "extra"} + + node_lvl1 = by_name["node_d2"].node + assert node_lvl1.provenance is not None + assert node_lvl1.provenance.metadata["type_name"] == "Node" + assert node_lvl1.provenance.metadata["component_id"] == hash(("jsonschema_ref", "Node")) + assert "inline_struct" not in node_lvl1.provenance.metadata + + node_lvl2 = next(f for f in node_lvl1.properties if f.name == "next").node + assert node_lvl2.type == "object" + assert node_lvl2.provenance is not None + assert node_lvl2.provenance.metadata["inline_struct"] is True + + assert by_name["opt_str"].node.type == "string" + assert by_name["opt_str"].node.nullable is True + + obj_u = by_name["obj_union"].node + assert obj_u.type == "object" + assert {f.name for f in obj_u.properties} == {"a", "b"} + assert obj_u.required_names == () diff --git a/tests/cdm_data_loaders/converters/sources/test_xsd.py b/tests/cdm_data_loaders/converters/sources/test_xsd.py index 841dbbd6..fc8ed414 100644 --- a/tests/cdm_data_loaders/converters/sources/test_xsd.py +++ b/tests/cdm_data_loaders/converters/sources/test_xsd.py @@ -3,7 +3,7 @@ import functools from collections.abc import Callable from pathlib import Path -from typing import Final +from typing import Any, Final import pyarrow as pa import pytest @@ -11,6 +11,7 @@ from pydantic import ValidationError from pyspark.sql.types import StructType +from cdm_data_loaders.converters.core.ir import Provenance, TypedNode from cdm_data_loaders.converters.emitters.dlt import DltEmitter from cdm_data_loaders.converters.emitters.json_schema.emitter import JsonSchemaEmitter from cdm_data_loaders.converters.emitters.pyarrow import PyArrowEmitter @@ -21,7 +22,12 @@ XsdToPyArrowError, ) from cdm_data_loaders.converters.facades.xsd_to_pyarrow import XsdToPyArrow -from cdm_data_loaders.converters.sources.xsd import XsdReader +from cdm_data_loaders.converters.sources.xsd import ( + XsdReader, + _string_ir_node, + _wrap_truncated_leaf_ir_field, + simple_type_to_ir_node, +) from cdm_data_loaders.schemas.uniref.schema import SCHEMAS as UNIREF_SCHEMAS from cdm_data_loaders.utils.xml_utils import local_name from cdm_data_loaders.validation.xsd.traversal import ( @@ -308,3 +314,180 @@ def test_xsd_reader_fail_invalid_options( """XsdReader validates max_recursion_depth and requires xml_tag when multiple roots exist.""" with pytest.raises(error_type, match=match): call() + + +IR_HELPERS_XSD: Final[str] = """ + + + List of ints + + + + Recursive type doc + + + + + + + Root element doc + + + + + + + + + +""" + +READER_IR_XSD: Final[str] = """ + + + List of ints + + + + Root element doc + + + + + + + + + +""" + + +@pytest.fixture +def ir_helpers_schema(tmp_path: Path) -> xmlschema.XMLSchema: + """Compile the XSD schema fixture used by IR helper tests.""" + xsd_path = tmp_path / "ir_helpers.xsd" + xsd_path.write_text(IR_HELPERS_XSD, encoding="utf-8") + return xmlschema.XMLSchema(str(xsd_path)) + + +@pytest.fixture +def reader_ir_schema(tmp_path: Path) -> xmlschema.XMLSchema: + """Compile the XSD schema fixture used by XsdReader provenance tests.""" + xsd_path = tmp_path / "reader_ir.xsd" + xsd_path.write_text(READER_IR_XSD, encoding="utf-8") + return xmlschema.XMLSchema(str(xsd_path)) + + +@pytest.mark.parametrize( + ("path", "nullable", "metadata", "expected_metadata"), + [ + pytest.param(("a",), True, {"k": "v"}, {"k": "v"}, id="nullable-with-metadata"), + pytest.param(("b",), False, None, {}, id="non-nullable-default-metadata"), + ], +) +def test_string_ir_node_pass( + path: tuple[str, ...], + *, + nullable: bool, + metadata: dict[str, Any] | None, + expected_metadata: dict[str, Any], +) -> None: + """_string_ir_node preserves nullable, provenance path, and metadata.""" + node = ( + _string_ir_node(path, nullable=nullable) + if metadata is None + else _string_ir_node(path, nullable=nullable, metadata=metadata) + ) + assert node.nullable is nullable + assert node.provenance == Provenance("xsd", path, expected_metadata) + + +@pytest.mark.parametrize( + ("type_kind", "path", "nullable", "expected_item_type", "expected_xsd_type"), + [ + pytest.param("builtin", ("attr",), False, "string", "xs:NMTOKEN", id="builtin-nmtokens"), + pytest.param("custom", ("ints",), True, "integer", "xs:integer", id="custom-int-list"), + ], +) +def test_simple_type_to_ir_node_pass_list_types( + ir_helpers_schema: xmlschema.XMLSchema, + type_kind: str, + path: tuple[str, ...], + *, + nullable: bool, + expected_item_type: str, + expected_xsd_type: str, +) -> None: + """simple_type_to_ir_node maps built-in and custom XSD list types with item provenance.""" + xsd_type = ( + ir_helpers_schema.elements["root"].type.attributes["req_attr"].type + if type_kind == "builtin" + else ir_helpers_schema.types["DocIntList"] + ) + node = simple_type_to_ir_node(xsd_type, path, nullable=nullable) + assert node.type == "array" + assert node.nullable is nullable + assert node.provenance == Provenance("xsd", path) + assert node.items is not None + assert node.items.type == expected_item_type + assert node.items.nullable is True + assert node.items.provenance == Provenance("xsd", (*path, "item"), {"xsd_type": expected_xsd_type}) + + +@pytest.mark.parametrize( + ("is_list", "nullable", "expected_required", "expected_node_type"), + [ + pytest.param(True, True, False, "array", id="list-nullable"), + pytest.param(False, False, True, "string", id="scalar-required"), + ], +) +def test_wrap_truncated_leaf_ir_field_pass( + ir_helpers_schema: xmlschema.XMLSchema, + *, + is_list: bool, + nullable: bool, + expected_required: bool, + expected_node_type: str, +) -> None: + """_wrap_truncated_leaf_ir_field attaches type documentation and truncated_component_id provenance.""" + rec_elem = ir_helpers_schema.elements["root"].type.content[1] + child_elem = rec_elem.type.content[1] + expected_meta = {"truncated_component_id": id(child_elem.type)} + + field = _wrap_truncated_leaf_ir_field(child_elem, ("rec", "child"), is_list=is_list, nullable=nullable) + assert field.name == "child" + assert field.required is expected_required + assert field.annotations == {"description": "Recursive type doc"} + assert field.provenance == Provenance("xsd", ("rec", "child"), expected_meta) + assert field.node.type == expected_node_type + assert field.node.nullable is nullable + assert field.node.provenance == Provenance("xsd", ("rec", "child"), expected_meta) + if is_list: + assert field.node.items is not None + assert field.node.items.nullable is True + assert field.node.items.provenance == Provenance("xsd", ("rec", "child", "item"), expected_meta) + + +def test_xsd_reader_read_pass_annotations_namespace_and_cache(reader_ir_schema: xmlschema.XMLSchema) -> None: + """XsdReader.read attaches element annotations, targetNamespace, provenance, and leaves struct_cache empty.""" + cache: dict[tuple[int, bool], TypedNode] = {} + doc = XsdReader(max_recursion_depth=1).read(reader_ir_schema, struct_cache=cache) + assert doc.name == "root" + assert doc.identifier == "urn:test:ir" + assert doc.annotations == {"description": "Root element doc"} + assert doc.root.annotations == {"description": "Root element doc"} + assert doc.provenance == Provenance("xsd", ("root",)) + fields_by_name = {f.name: f for f in doc.root.properties} + assert fields_by_name["_req_attr"].required is True + assert fields_by_name["ints"].annotations == {"description": "List of ints"} + assert fields_by_name["ints"].provenance == Provenance("xsd", ("ints",)) + assert fields_by_name["val"].provenance == Provenance("xsd", ("val",), {"xsd_type": "xs:string"}) + assert cache == {} + + +def test_xsd_reader_read_at_level_pass_scalar_leaf_provenance(reader_ir_schema: xmlschema.XMLSchema) -> None: + """XsdReader.read_at_level marks scalar leaf tables with scalar_leaf_table provenance and #text property.""" + level_docs = XsdReader().read_at_level(reader_ir_schema, 1, root_tag="root") + assert "val" in level_docs + assert level_docs["val"].root.provenance == Provenance("xsd", (), {"scalar_leaf_table": True}) + assert level_docs["val"].root.properties[0].name == "#text" diff --git a/tests/cdm_data_loaders/core/test_buffer.py b/tests/cdm_data_loaders/core/test_buffer.py index 74120e9a..5e8cf66e 100644 --- a/tests/cdm_data_loaders/core/test_buffer.py +++ b/tests/cdm_data_loaders/core/test_buffer.py @@ -1,6 +1,9 @@ """Tests for the page buffer in cdm_data_loaders.core.buffer.""" +import json from collections.abc import Iterator +from pathlib import Path +from types import SimpleNamespace from typing import Any, Final import dlt @@ -15,6 +18,7 @@ DictBuffer, format_rejected_records, make_rejected_row, + make_rejected_table_result, resolve_destination_schema, ) from cdm_data_loaders.core.fields import REJECTED_TABLE_NAME @@ -447,18 +451,67 @@ def test_resolve_destination_schema_resolves_rejected_and_mapped_tables( assert resolve_destination_schema(table_name, schema_map) == expected_schema -@pytest.mark.parametrize("use_pyarrow", [True, False], ids=["pyarrow-page", "dict-list-page"]) -def test_format_rejected_records_formats_pyarrow_and_dict_pages(use_pyarrow: bool) -> None: +@pytest.mark.parametrize( + "use_pyarrow", + [ + pytest.param(None, id="default-pyarrow-page"), + pytest.param(True, id="pyarrow-page"), + pytest.param(False, id="dict-list-page"), + ], +) +def test_format_rejected_records_formats_pyarrow_and_dict_pages(use_pyarrow: bool | None) -> None: """format_rejected_records wraps rejected rows with _rejected table metadata as PyArrow or list pages.""" rows = [ make_rejected_row("people", "bad row", source_file="people.csv", line_no=2, raw_record="x"), ] - item = format_rejected_records(rows, use_pyarrow=use_pyarrow) + item = ( + format_rejected_records(rows) if use_pyarrow is None else format_rejected_records(rows, use_pyarrow=use_pyarrow) + ) assert item.meta.table_name == REJECTED_TABLE_NAME - if use_pyarrow: + if use_pyarrow is False: + assert item.data == rows + else: assert isinstance(item.data, pa.Table) assert item.data.schema.equals(REJECTED_PYARROW_SCHEMA) assert item.data.to_pylist() == rows - else: - assert item.data == rows + + +def test_make_rejected_table_result_serializes_non_json_values() -> None: + """make_rejected_table_result serializes non-JSON values via str and wraps the row under _rejected.""" + expected_line_no = 42 + result = make_rejected_table_result( + "people", + {"p": Path("a.txt")}, + "bad", + source_file="f.jsonl", + line_no=expected_line_no, + ) + + assert result == { + REJECTED_TABLE_NAME: [ + { + "table_name": "people", + "source_file": "f.jsonl", + "line_no": expected_line_no, + "raw_record": json.dumps({"p": "a.txt"}), + "error_detail": "bad", + } + ] + } + + +def test_dict_buffer_from_settings_preserves_subclass_and_defaults() -> None: + """DictBuffer.from_settings instantiates the subclass and defaults missing optional settings.""" + + class _SubDictBuffer(DictBuffer): + pass + + expected_max_items = 5 + minimal_settings = SimpleNamespace(buffer_size=expected_max_items) + sub_buf = _SubDictBuffer.from_settings(minimal_settings) # type: ignore[arg-type] + + assert isinstance(sub_buf, _SubDictBuffer) + assert sub_buf.max_items == expected_max_items + assert sub_buf.schema_map is None + assert sub_buf.use_pyarrow is True diff --git a/tests/cdm_data_loaders/core/test_fields.py b/tests/cdm_data_loaders/core/test_fields.py index 51b4e4d9..655ea186 100644 --- a/tests/cdm_data_loaders/core/test_fields.py +++ b/tests/cdm_data_loaders/core/test_fields.py @@ -166,7 +166,7 @@ class TableNameModel(BaseModel): def test_table_name_fail_reserved_rejected_table_name() -> None: """TableName rejects the reserved _rejected table name.""" assert REJECTED_TABLE_NAME == "_rejected" - with pytest.raises(ValidationError, match=REJECTED_TABLE_NAME): + with pytest.raises(ValidationError, match=f"Table name {REJECTED_TABLE_NAME!r} is reserved"): TableNameModel.model_validate({"table_name": REJECTED_TABLE_NAME}) @@ -262,6 +262,11 @@ def test_file_glob_fail_empty_string() -> None: f"Glob pattern for XML files. {CTS_GLOB_WILDCARDS_DESCRIPTION}", id="with_trailing_period", ), + pytest.param( + "Glob pattern for suffix X.", + f"Glob pattern for suffix X. {CTS_GLOB_WILDCARDS_DESCRIPTION}", + id="preserves_non_period_trailing_chars", + ), ], ) def test_file_glob_description_pass(summary: str, expected: str) -> None: diff --git a/tests/cdm_data_loaders/core/test_file_items.py b/tests/cdm_data_loaders/core/test_file_items.py index 36d3929a..51534f46 100644 --- a/tests/cdm_data_loaders/core/test_file_items.py +++ b/tests/cdm_data_loaders/core/test_file_items.py @@ -1,8 +1,11 @@ """Tests for the FileItemDict helpers in cdm_data_loaders.core.file_items.""" +import errno import gzip import io +import os from pathlib import Path +from types import SimpleNamespace from typing import Final import fsspec @@ -95,14 +98,20 @@ def test_as_file_item_fail_missing_file(tmp_path: Path) -> None: with pytest.raises(FileNotFoundError, match="No such file or directory") as exc_info: as_file_item(missing) + assert exc_info.value.errno == errno.ENOENT + assert exc_info.value.strerror == os.strerror(errno.ENOENT) assert exc_info.value.filename == str(missing) def test_as_file_item_fail_directory(tmp_path: Path) -> None: - """Verify a directory raises IsADirectoryError.""" - with pytest.raises(IsADirectoryError): + """Verify a directory raises IsADirectoryError with errno, strerror, and filename.""" + with pytest.raises(IsADirectoryError) as exc_info: as_file_item(tmp_path) + assert exc_info.value.errno == errno.EISDIR + assert exc_info.value.strerror == os.strerror(errno.EISDIR) + assert exc_info.value.filename == str(tmp_path) + """join_url""" @@ -169,6 +178,24 @@ def test_source_url_pass_file_item(fs_backend: FsBackend) -> None: assert source_url(item) == item["file_url"] +@pytest.mark.parametrize( + "named_handle", + [ + pytest.param(True, id="named-file-handle"), + pytest.param(False, id="unnamed-bytes-io"), + ], +) +def test_source_url_pass_open_handle(tmp_path: Path, named_handle: bool) -> None: + """Verify source_url uses .name on a file handle and falls back to str(handle) when unnamed.""" + if named_handle: + path = _write(tmp_path, "open.tsv", CONTENT) + with path.open("rb") as fh: + assert source_url(fh) == str(path) + else: + buf = io.BytesIO(CONTENT) + assert source_url(buf) == str(buf) + + """open_binary""" @@ -217,6 +244,16 @@ def test_is_local_pass_matches_backend(fs_backend: FsBackend) -> None: assert is_local(fs_backend.file_item("data.tsv")) is fs_backend.is_local +def test_is_local_pass_string_protocol() -> None: + """Verify is_local handles filesystems whose protocol attribute is a plain string.""" + item = FileItemDict( + {"file_name": "a.tsv", "relative_path": "a.tsv", "file_url": "file:///a.tsv", "mime_type": "text/tsv"}, + SimpleNamespace(protocol="file"), # type: ignore[arg-type] + ) + + assert is_local(item) is True + + """local_copy""" diff --git a/tests/cdm_data_loaders/core/test_filesystem.py b/tests/cdm_data_loaders/core/test_filesystem.py index c94ebccd..78766f50 100644 --- a/tests/cdm_data_loaders/core/test_filesystem.py +++ b/tests/cdm_data_loaders/core/test_filesystem.py @@ -294,7 +294,7 @@ def test_collect_file_listing_fail_without_filesystem_resource( ) with pytest.raises( ValueError, - match="loader_file_format='file_listing' requires at least one dlt filesystem resource", + match=r"^loader_file_format='file_listing' requires at least one dlt filesystem resource\.$", ): collect_file_listing(settings, invalid_resource) @@ -367,6 +367,22 @@ def read_none(_items: TDataItems, _cfg: InputLocalOutputSettings) -> None: assert collector.preprocess_errors == [] +def test_attach_file_collector_pass_repeated_attach_on_plain_filesystem_rebinds_tracker(tmp_path: Path) -> None: + """Calling attach_file_collector twice on a plain filesystem resource rebinds the tracker without double-mapping.""" + (tmp_path / "only.txt").write_text("hello", encoding="utf-8") + fs_resource = filesystem(bucket_url=str(tmp_path), file_glob="*.txt") + + first_collector = attach_file_collector(fs_resource) + second_collector = attach_file_collector(fs_resource) + + items = list(fs_resource) + assert [item["relative_path"] for item in items] == ["only.txt"] + assert first_collector.ingested_files == [] + assert first_collector.preprocess_errors == [] + assert second_collector.ingested_files == ["only.txt"] + assert second_collector.preprocess_errors == [] + + def test_build_filesystem_resource_pass_default_glob_and_callable_transformer( input_dir: Path, output_dir: Path ) -> None: @@ -842,3 +858,28 @@ class PresetIngestSettings(FileFormatIngestSettings, cli_prog_name="preset_inges }, "pipeline_run_kwargs": {"loader_file_format": "parquet"}, } + + +def test_build_file_ingest_kwargs_defaults_fallback_dataset_name_to_false(tmp_path: Path) -> None: + """build_file_ingest_kwargs defaults fallback_dataset_name to False and preserves dataset_name=None.""" + settings = make_cts_settings( + FileFormatIngestSettings, + tmp_path, + dataset_name="unused_ds", + loader_file_format="parquet", + ).model_copy(update={"dataset_name": None}) + + result = build_file_ingest_kwargs(settings, TINY_PIPELINE_NAME) + + assert result == { + "pipeline_kwargs": { + "pipeline_name": TINY_PIPELINE_NAME, + "dataset_name": None, + }, + "pipeline_run_kwargs": {"loader_file_format": "parquet"}, + } + + +def test_collect_relative_paths_flattens_batched_lists() -> None: + """_collect_relative_paths extracts relative_path values from list-batched file items.""" + assert _collect_relative_paths([[{"relative_path": "a.txt"}, {"relative_path": "b.txt"}]]) == ["a.txt", "b.txt"] diff --git a/tests/cdm_data_loaders/core/test_md5_preprocessor.py b/tests/cdm_data_loaders/core/test_md5_preprocessor.py index ecdaef0c..7801cbdd 100644 --- a/tests/cdm_data_loaders/core/test_md5_preprocessor.py +++ b/tests/cdm_data_loaders/core/test_md5_preprocessor.py @@ -1,18 +1,21 @@ """Unit tests for ``cdm_data_loaders.core.md5_preprocessor``.""" import gzip +from pathlib import Path from typing import Final from unittest.mock import patch import pytest +from cdm_data_loaders.core import md5_preprocessor +from cdm_data_loaders.core.file_items import as_file_item from cdm_data_loaders.core.md5_preprocessor import ( FILE_PREPROCESSORS, parse_md5_sidecar, validate_md5_sidecar, ) from cdm_data_loaders.core.preprocessors import FilePreprocessError -from cdm_data_loaders.utils.checksums import new_hasher +from cdm_data_loaders.utils.checksums import HASH_CHUNK_BYTES, new_hasher from tests.helpers.fs_backends import FsBackend SAMPLE_DIGEST: Final[str] = "d41d8cd98f00b204e9800998ecf8427e" @@ -150,3 +153,27 @@ def test_validate_md5_sidecar_fail_oserror_reading_sidecar(fs_backend: FsBackend ), ): validate_md5_sidecar(file_item) + + +def test_validate_md5_sidecar_forwards_open_and_checksum_arguments(tmp_path: Path) -> None: + """validate_md5_sidecar opens sidecar as UTF-8 text, raw stream binary uncompressed, and passes HASH_CHUNK_BYTES.""" + gz_path = tmp_path / "sample.xml.gz" + raw = gzip.compress(b"", mtime=0) + gz_path.write_bytes(raw) + (tmp_path / "sample.xml.gz.md5").write_text(f"{_md5_hex(raw)} sample.xml.gz\n", encoding="utf-8") + file_item = as_file_item(gz_path) + + with ( + patch.object(file_item.fsspec, "open", wraps=file_item.fsspec.open) as mock_fs_open, + patch.object(file_item, "open", wraps=file_item.open) as mock_item_open, + patch.object( + md5_preprocessor, + "compute_stream_checksum", + wraps=md5_preprocessor.compute_stream_checksum, + ) as mock_checksum, + ): + validate_md5_sidecar(file_item) + + assert mock_fs_open.call_args_list[0].kwargs == {"mode": "rt", "encoding": "utf-8"} + assert mock_item_open.call_args.kwargs == {"mode": "rb", "compression": "disable"} + assert mock_checksum.call_args.kwargs["chunk_size"] == HASH_CHUNK_BYTES diff --git a/tests/cdm_data_loaders/core/test_pipeline_catalog.py b/tests/cdm_data_loaders/core/test_pipeline_catalog.py index 34af98c0..81f22f57 100644 --- a/tests/cdm_data_loaders/core/test_pipeline_catalog.py +++ b/tests/cdm_data_loaders/core/test_pipeline_catalog.py @@ -1,16 +1,23 @@ """Unit tests for ``cdm_data_loaders.core.pipeline_catalog``.""" +import re +import sys from pathlib import Path from types import ModuleType, SimpleNamespace from unittest.mock import patch import pytest +from pydantic_settings import SettingsConfigDict from cdm_data_loaders.core import pipeline_catalog from cdm_data_loaders.core.cli import CliRunner from cdm_data_loaders.core.pipeline_catalog import ( PACKAGE_DIST_NAME, PipelineSpec, + _build_module_to_cli_map, + _resolve_callable_spec, + _resolve_runner_spec, + _spec_from_module, discover_pipeline_targets, load_project_scripts, resolve_pipeline, @@ -126,3 +133,325 @@ def test_resolve_pipeline_short_name_and_unregistered_runner( ) assert resolve_pipeline(target, pyproject_path=pyproject_path) == expected_spec + + +@pytest.mark.parametrize( + ("script_spec", "expected_cli_command"), + [ + pytest.param( + "cdm_data_loaders.pipelines.jsonlines.extract:cli", + "jsonlines_extract", + id="registered-with-attr", + ), + pytest.param( + "cdm_data_loaders.pipelines.jsonlines.extract", + "jsonlines_extract", + id="registered-without-attr", + ), + pytest.param( + "cdm_data_loaders.pipelines.jsonlines.extract:cli", + "custom_extract", + id="registered-custom-command", + ), + ], +) +def test_resolve_callable_spec_registered_module( + script_spec: str, + expected_cli_command: str, +) -> None: + """Resolve a callable whose module is registered in module_to_cli.""" + module_path = "cdm_data_loaders.pipelines.jsonlines.extract" + scripts = {expected_cli_command: script_spec} + module_to_cli = {module_path: expected_cli_command} + expected_spec = PipelineSpec( + cli_command=expected_cli_command, + module_path=module_path, + settings_cls=JsonlExtractSettings, + pipeline_fn=run_jsonlines_pipeline, + cli_callable=jsonl_extract_cli, + ) + + assert _resolve_callable_spec(run_jsonlines_pipeline, scripts, module_to_cli) == expected_spec + + +def test_resolve_callable_spec_imports_unloaded_module( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Import the callable's module when it is not yet in sys.modules.""" + module_path = "cdm_data_loaders.pipelines.jsonlines.extract" + original_mod = sys.modules.get(module_path) + monkeypatch.delitem(sys.modules, module_path, raising=False) + scripts = {"jsonlines_extract": f"{module_path}:cli"} + module_to_cli = {module_path: "jsonlines_extract"} + + result = _resolve_callable_spec(run_jsonlines_pipeline, scripts, module_to_cli) + reloaded_mod = sys.modules[module_path] + + assert reloaded_mod is not original_mod + assert result == PipelineSpec( + cli_command="jsonlines_extract", + module_path=module_path, + settings_cls=reloaded_mod.JsonlExtractSettings, + pipeline_fn=reloaded_mod.run_jsonlines_pipeline, + cli_callable=reloaded_mod.cli, + ) + + +def test_resolve_callable_spec_unregistered_module_cli_runner( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Fall back to module.cli when module_path is unregistered and cli.action_fn matches.""" + mod_name = "tests_synthetic_unregistered_runner_mod" + + def action_fn(_settings: JsonlExtractSettings) -> None: + return None + + action_fn.__module__ = mod_name + runner = CliRunner(JsonlExtractSettings, action_fn) + mod = ModuleType(mod_name) + mod.cli = runner # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, mod_name, mod) + + result = _resolve_callable_spec(action_fn, scripts={}, module_to_cli={}) + + assert result == PipelineSpec( + cli_command="jsonlines_extract", + module_path=mod_name, + settings_cls=JsonlExtractSettings, + pipeline_fn=action_fn, + cli_callable=runner, + ) + + +@pytest.mark.parametrize( + "case", + ["no_module", "no_cli_attr", "non_runner_cli_attr", "different_action_fn"], + ids=["no_module", "no_cli_attr", "non_runner_cli_attr", "different_action_fn"], +) +def test_resolve_callable_spec_rejects_unresolvable_callable( + monkeypatch: pytest.MonkeyPatch, + case: str, +) -> None: + """Raise ValueError with callable and module context when a callable cannot be resolved.""" + if case == "no_module": + + def no_mod_fn(_settings: JsonlExtractSettings) -> None: + return None + + no_mod_fn.__module__ = "" + fn: object = no_mod_fn + expected_mod = "" + else: + mod_name = f"tests_synthetic_unresolvable_{case}" + + def target_fn(_settings: JsonlExtractSettings) -> None: + return None + + def other_fn(_settings: JsonlExtractSettings) -> None: + return None + + target_fn.__module__ = mod_name + mod = ModuleType(mod_name) + if case == "non_runner_cli_attr": + mod.cli = other_fn # type: ignore[attr-defined] + elif case == "different_action_fn": + mod.cli = CliRunner(JsonlExtractSettings, other_fn) # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, mod_name, mod) + fn = target_fn + expected_mod = mod_name + + expected_msg = f"Cannot resolve pipeline from callable {fn!r} (module {expected_mod!r})" + with pytest.raises(ValueError, match=re.escape(expected_msg)): + _resolve_callable_spec(fn, scripts={}, module_to_cli={}) # type: ignore[arg-type] + + +def test_load_project_scripts_defaults_to_repo_pyproject() -> None: + """Read repository pyproject.toml when pyproject_path is None.""" + default_scripts = load_project_scripts() + assert "xsv" in default_scripts + + +def test_load_project_scripts_returns_empty_when_scripts_missing(tmp_path: Path) -> None: + """Return an empty mapping when [project.scripts] is absent from pyproject.toml.""" + empty_toml = tmp_path / "empty.toml" + empty_toml.write_text("[project]\nname = 'demo'\n", encoding="utf-8") + + assert load_project_scripts(empty_toml) == {} + assert discover_pipeline_targets(empty_toml) == {} + + +def test_build_module_to_cli_map_prefers_cli_attr_and_handles_no_colon() -> None: + """Prefer :cli over alphabetically earlier aliases and handle specs without a colon.""" + scripts = { + "aaa_alias": "pkg.mod:other", + "zzz_primary": "pkg.mod:cli", + "bare": "pkg.bare", + } + assert _build_module_to_cli_map(scripts) == { + "pkg.mod": "zzz_primary", + "pkg.bare": "bare", + } + + +@pytest.mark.parametrize( + "case", + [ + pytest.param("explicit_runner", id="explicit-runner-attr"), + pytest.param("prefixed_plain_callable", id="command-prefix-plain-callable"), + pytest.param("non_callable_attr", id="non-callable-attr"), + ], +) +def test_spec_from_module_resolves_cli_attribute_variants(case: str) -> None: + """Resolve explicit attr_name, _cli fallback, and non-callable attributes on a module.""" + mod = ModuleType("tests_synthetic_spec_mod") + runner = CliRunner(JsonlExtractSettings, run_jsonlines_pipeline) + + def plain_cli() -> None: + return None + + mod.custom_cli = runner # type: ignore[attr-defined] + mod.plain_cli = plain_cli # type: ignore[attr-defined] + mod.not_callable = 42 # type: ignore[attr-defined] + mod.Settings = JsonlExtractSettings # type: ignore[attr-defined] + mod.run_pipeline = run_jsonlines_pipeline # type: ignore[attr-defined] + + if case == "explicit_runner": + result = _spec_from_module("ignored", mod, attr_name="custom_cli") + expected = PipelineSpec( + cli_command="ignored", + module_path="tests_synthetic_spec_mod", + settings_cls=JsonlExtractSettings, + pipeline_fn=run_jsonlines_pipeline, + cli_callable=runner, + ) + elif case == "prefixed_plain_callable": + result = _spec_from_module("plain", mod) + expected = PipelineSpec( + cli_command="plain", + module_path="tests_synthetic_spec_mod", + settings_cls=JsonlExtractSettings, + pipeline_fn=run_jsonlines_pipeline, + cli_callable=plain_cli, + ) + else: + result = _spec_from_module("other", mod, attr_name="not_callable") + expected = PipelineSpec( + cli_command="other", + module_path="tests_synthetic_spec_mod", + settings_cls=JsonlExtractSettings, + pipeline_fn=run_jsonlines_pipeline, + cli_callable=None, + ) + + assert result == expected + + +def test_resolve_runner_spec_falls_back_to_module_stem_without_cli_prog_name() -> None: + """Derive cli_command from the action module stem when cli_prog_name is None.""" + + class NoProgSettings(JsonlExtractSettings): + model_config = SettingsConfigDict(**{**JsonlExtractSettings.model_config, "cli_prog_name": None}) + + def dummy_action(_s: NoProgSettings) -> None: + return None + + dummy_action.__module__ = "pkg.sub.my_pipe" + no_prog_runner = CliRunner(NoProgSettings, dummy_action) + + result = _resolve_runner_spec(no_prog_runner, scripts={}, module_to_cli={}) + + assert result == PipelineSpec( + cli_command="my_pipe", + module_path="pkg.sub.my_pipe", + settings_cls=NoProgSettings, + pipeline_fn=dummy_action, + cli_callable=no_prog_runner, + ) + + +@pytest.mark.parametrize( + "target_case", + [ + pytest.param("registered_runner", id="registered-cli-runner"), + pytest.param("registered_module", id="registered-module"), + pytest.param("unregistered_module_with_cli", id="unregistered-module-with-cli"), + pytest.param("registered_callable", id="registered-callable"), + ], +) +def test_resolve_pipeline_registered_and_fallback_targets( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + target_case: str, +) -> None: + """Resolve registered CliRunner, module, callable, and cli-bearing unregistered module targets.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + '[project.scripts]\njsonlines_extract = "cdm_data_loaders.pipelines.jsonlines.extract:cli"\n', + encoding="utf-8", + ) + expected_module_path = "cdm_data_loaders.pipelines.jsonlines.extract" + + if target_case == "registered_runner": + target: object = jsonl_extract_cli + elif target_case == "registered_module": + target = sys.modules[expected_module_path] + elif target_case == "registered_callable": + target = run_jsonlines_pipeline + else: + mod_with_cli = ModuleType("cdm_data_loaders.pipelines.custom.entry") + mod_with_cli.cli = jsonl_extract_cli # type: ignore[attr-defined] + monkeypatch.setitem(sys.modules, mod_with_cli.__name__, mod_with_cli) + target = mod_with_cli + expected_module_path = mod_with_cli.__name__ + + result = resolve_pipeline(target, pyproject_path=pyproject) # type: ignore[arg-type] + + assert result == PipelineSpec( + cli_command="jsonlines_extract", + module_path=expected_module_path, + settings_cls=JsonlExtractSettings, + pipeline_fn=run_jsonlines_pipeline, + cli_callable=jsonl_extract_cli, + ) + + +def test_resolve_pipeline_returns_pipeline_spec_unchanged() -> None: + """Return an existing PipelineSpec instance unchanged.""" + spec = PipelineSpec(cli_command="demo", module_path="pkg.demo") + assert resolve_pipeline(spec) is spec + + +@pytest.mark.parametrize( + "target", + [ + pytest.param(" jsonlines_extract ", id="whitespace-padded"), + pytest.param("cdm_data_loaders.pipelines.jsonlines_extract", id="prefixed-command"), + pytest.param("cdm_data_loaders.pipelines.jsonlines.extract", id="full-module-path"), + ], +) +def test_resolve_pipeline_resolves_string_variants(target: str) -> None: + """Resolve whitespace-padded, package-prefixed, and full module path string targets.""" + result = resolve_pipeline(target) + assert result.cli_command == "jsonlines_extract" + + +@pytest.mark.parametrize( + ("target", "expected_match"), + [ + pytest.param("nope", r"Unknown pipeline 'nope'\. Known pipelines: alpha, beta", id="unknown-string"), + pytest.param(123, r"Unknown pipeline 123\. Known pipelines: alpha, beta", id="unsupported-type"), + ], +) +def test_resolve_pipeline_fail_unknown_target( + tmp_path: Path, + target: object, + expected_match: str, +) -> None: + """Raise ValueError listing known pipelines for unknown or unsupported targets.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + '[project.scripts]\nbeta = "pkg.b:cli"\nalpha = "pkg.a:cli"\n', + encoding="utf-8", + ) + with pytest.raises(ValueError, match=expected_match): + resolve_pipeline(target, pyproject_path=pyproject) # type: ignore[arg-type] diff --git a/tests/cdm_data_loaders/core/test_reporting.py b/tests/cdm_data_loaders/core/test_reporting.py index 1108997c..ce0edb16 100644 --- a/tests/cdm_data_loaders/core/test_reporting.py +++ b/tests/cdm_data_loaders/core/test_reporting.py @@ -20,6 +20,7 @@ ParsedLoadJobFileName, TPackageJobState, ) +from dlt.extract.resource import DltResource from pydantic import ValidationError from requests import RequestException @@ -1116,3 +1117,43 @@ def test_write_pipeline_report_pass_datetime_subclass_without_global_safe_dumper "pipeline_load_info_errs": None, }, ) + + +def test_save_pipeline_metadata_sets_text_columns_on_load_info_resource(tmp_path: Path) -> None: + """save_pipeline_metadata configures text data_type on nullable metadata columns of _load_info.""" + captured_resources: list[DltResource] = [] + + class _PipelineStub: + pipeline_name = TEST_PIPELINE_NAME + dataset_name = TEST_DATASET_NAME + last_trace = SimpleNamespace(started_at=MOCK_START_TIME, finished_at=MOCK_FINISH_TIME) + + def run(self, resource: DltResource, **_kwargs: object) -> SimpleNamespace: + captured_resources.append(resource) + return SimpleNamespace(has_failed_jobs=False) + + in_dir = tmp_path / "in" + in_dir.mkdir() + settings = CtsSettings( + input_dir=str(in_dir), + output_dir=tmp_path / "out", + use_destination="local_fs", + ) + (tmp_path / "out" / PIPELINE_DIR_NAME).mkdir(parents=True, exist_ok=True) + load_info = create_load_info_one_completed_two_failed() + + save_pipeline_metadata(settings, _PipelineStub(), load_info, err_msg=None) # type: ignore[arg-type] + + assert len(captured_resources) == 1 + resource = captured_resources[0] + assert resource.max_table_nesting == 0 + assert resource.columns == { + k: {"name": k, "data_type": "text"} + for k in [ + "environment", + "staging_type", + "staging_name", + "staging_displayable_credentials", + "dataset_name", + ] + } diff --git a/tests/cdm_data_loaders/core/test_settings_check_aliases.py b/tests/cdm_data_loaders/core/test_settings_check_aliases.py index 9a0b50cc..daa8f706 100644 --- a/tests/cdm_data_loaders/core/test_settings_check_aliases.py +++ b/tests/cdm_data_loaders/core/test_settings_check_aliases.py @@ -113,6 +113,12 @@ def test_check_aliases_pass_valid_names( "MySettingsClass: CLI name 'beta' is claimed by both 'alpha' and 'beta'", id="alias_is_other_field_name", ), + pytest.param( + {"alpha": Field("a", alias="beta", validation_alias="gamma"), "beta": "b"}, + None, + "MySettingsClass: CLI name 'beta' is claimed by both 'alpha' and 'beta'", + id="alias_distinct_from_validation_alias_collides", + ), pytest.param( {"output_dir": "x"}, {"output-dir": "output-dir"}, @@ -151,8 +157,8 @@ def test_check_aliases_pass_valid_names( ), pytest.param( {"alpha": "a"}, - {"alpha": ["-a", "h"], "nope": "n"}, - "MySettingsClass: invalid shortcut '-a' for 'alpha'; omit the leading dashes\nMySettingsClass: CLI name 'h' for 'alpha' is reserved by argparse\nMySettingsClass: cli_shortcuts target 'nope' does not match any CLI argument", + {"nope": "n", "alpha": ["-a", "h"]}, + "MySettingsClass: cli_shortcuts target 'nope' does not match any CLI argument\nMySettingsClass: invalid shortcut '-a' for 'alpha'; omit the leading dashes\nMySettingsClass: CLI name 'h' for 'alpha' is reserved by argparse", id="all_errors_reported", ), ], diff --git a/tests/cdm_data_loaders/core/test_transforms.py b/tests/cdm_data_loaders/core/test_transforms.py index 2ee32a61..f8e4ada2 100644 --- a/tests/cdm_data_loaders/core/test_transforms.py +++ b/tests/cdm_data_loaders/core/test_transforms.py @@ -4,7 +4,7 @@ from collections.abc import Iterable from pathlib import Path from types import SimpleNamespace -from typing import Annotated, Any, Final +from typing import Annotated, Any, Final, Union import pyarrow as pa import pytest @@ -23,14 +23,21 @@ TransformRegistry, ) from cdm_data_loaders.core.transforms.introspection import ( + _annotation_capabilities, + _annotation_is_transform_context, + _safe_type_hints, + _unwrap_partial_target, accepts_pyarrow_table, accepts_record_dict, + inspect_transform_signature, ) from cdm_data_loaders.core.transforms.routing import ( align_and_cast_table, compose_transforms, emit_raw_table, + invoke_transform, make_record_router, + merge_routed_outputs, resolve_schema_map, route_and_coerce_record, route_and_coerce_table, @@ -38,6 +45,7 @@ ) from cdm_data_loaders.core.transforms.settings_mixin import ( LockedTransformHooksMixin, + TransformSettingsMixin, reconcile_multi_table_schema_map, validate_table_transforms, ) @@ -860,7 +868,7 @@ def test_emit_raw_table_pass_arrow_and_dict( def test_emit_raw_table_fail_empty_prefix() -> None: """emit_raw_table raises ValueError when prefix is empty.""" ctx = TransformContext.for_source("id_mapping", "dir/sample.dat") - with pytest.raises(ValueError, match="prefix must be a non-empty string"): + with pytest.raises(ValueError, match=r"^prefix must be a non-empty string$"): emit_raw_table({"id": "1"}, ctx, prefix="") @@ -1280,35 +1288,78 @@ class LockedExtractSettings(LockedTransformHooksMixin, JsonlExtractSettings, cli @pytest.mark.parametrize( - ("use_pyarrow", "transformed_tables", "explicit_schemas", "expected_built", "expected_tables"), + ( + "use_pyarrow", + "transformed_tables", + "explicit_schemas", + "none_default_tables", + "expected_built", + "expected_registry", + ), [ - pytest.param(False, (), {}, [], None, id="use-pyarrow-false"), - pytest.param(True, (), {}, ["alpha", "beta"], {"alpha", "beta"}, id="all-defaults"), - pytest.param(True, ("alpha",), {}, ["beta"], {"beta"}, id="skips-transformed-without-explicit-schema"), + pytest.param(False, (), {}, (), [], None, id="use-pyarrow-false"), + pytest.param( + True, + (), + {}, + (), + ["alpha", "beta"], + {"alpha": ITEMS_SCHEMA, "beta": ITEMS_SCHEMA}, + id="all-defaults", + ), + pytest.param( + True, + ("alpha",), + {}, + (), + ["beta"], + {"beta": ITEMS_SCHEMA}, + id="skips-transformed-without-explicit-schema", + ), pytest.param( True, ("alpha",), {"alpha_out": SOURCED_ITEMS_SCHEMA}, + (), ["beta"], - {"alpha_out", "beta"}, + {"alpha_out": SOURCED_ITEMS_SCHEMA, "beta": ITEMS_SCHEMA}, id="merges-explicit-fanout-and-untransformed-default", ), - pytest.param(True, ("alpha", "beta"), {}, [], None, id="all-transformed-unschematized"), + pytest.param( + True, + (), + {"alpha": OVERRIDE_SCHEMA}, + (), + ["beta"], + {"alpha": OVERRIDE_SCHEMA, "beta": ITEMS_SCHEMA}, + id="explicit-source-table-override-skips-default-builder", + ), + pytest.param( + True, + (), + {}, + ("alpha",), + ["alpha", "beta"], + {"beta": ITEMS_SCHEMA}, + id="skips-none-default-schema", + ), + pytest.param(True, ("alpha", "beta"), {}, (), [], None, id="all-transformed-unschematized"), ], ) def test_reconcile_multi_table_schema_map_reconciles_defaults_and_overrides( use_pyarrow: bool, transformed_tables: tuple[str, ...], explicit_schemas: dict[str, pa.Schema], + none_default_tables: tuple[str, ...], expected_built: list[str], - expected_tables: set[str] | None, + expected_registry: dict[str, pa.Schema] | None, ) -> None: """reconcile_multi_table_schema_map builds defaults only for untransformed unmapped tables and merges explicit schemas.""" built: list[str] = [] def _build_default(table_name: str) -> pa.Schema | None: built.append(table_name) - return ITEMS_SCHEMA + return None if table_name in none_default_tables else ITEMS_SCHEMA transforms = ( TransformRegistry(registry=dict.fromkeys(transformed_tables, _identity_items_transform)) @@ -1326,8 +1377,313 @@ def _build_default(table_name: str) -> pa.Schema | None: ) assert built == expected_built - if expected_tables is None: - assert result is None + expected = ArrowSchemaRegistry(registry=expected_registry) if expected_registry is not None else None + assert result == expected + + +type _CtxAlias = TransformContext + + +def test_unwrap_partial_target_unwraps_nested_partials() -> None: + """_unwrap_partial_target unwraps arbitrarily nested functools.partial wrappers.""" + + def base_fn(table: pa.Table, *, _flag: bool = False) -> dict[str, Any]: + del table, _flag + return {} + + nested = functools.partial(functools.partial(base_fn, _flag=True)) + assert _unwrap_partial_target(nested) is base_fn + + +@pytest.mark.parametrize( + "case", + [ + pytest.param("nested_partial", id="nested-partial"), + pytest.param("callable_object", id="callable-object"), + pytest.param("unresolvable_annotation", id="unresolvable-annotation"), + ], +) +def test_safe_type_hints_handles_callables_and_errors(case: str) -> None: + """_safe_type_hints resolves hints on partials and callable objects and returns empty dict on errors.""" + if case == "nested_partial": + + def base_fn(table: pa.Table, *, _flag: bool = False) -> dict[str, Any]: + del table, _flag + return {} + + target: object = functools.partial(functools.partial(base_fn, _flag=True)) + assert _safe_type_hints(target)["table"] is pa.Table # type: ignore[arg-type] + elif case == "callable_object": + + class _CallableObj: + def __call__(self, table: pa.Table, ctx: TransformContext | None = None) -> dict[str, Any]: + del table, ctx + return {} + + assert _safe_type_hints(_CallableObj())["table"] is pa.Table + else: + + def bad_hint_fn(_x: object) -> dict[str, Any]: + return {} + + bad_hint_fn.__annotations__["_x"] = "MissingLocalType" + assert _safe_type_hints(bad_hint_fn) == {} + + +@pytest.mark.parametrize( + ("ann", "expected_caps"), + [ + pytest.param(pa.Table, (True, False), id="pa_table"), + pytest.param("pa.Table", (True, False), id="str_table"), + pytest.param("pa.Table | dict[str, Any]", (True, True), id="str_table_or_dict"), + pytest.param("pa.Table | Mapping[str, Any]", (True, True), id="str_table_or_mapping"), + pytest.param("dict[str, Any]", (False, True), id="str_dict"), + pytest.param(pa.Table | None, (True, False), id="optional_pa_table"), + pytest.param(Annotated[pa.Table, "meta"], (True, False), id="annotated_pa_table"), + pytest.param(int, (False, True), id="fallback_int"), + ], +) +def test_annotation_capabilities_classifies_types(ann: object, expected_caps: tuple[bool, bool]) -> None: + """_annotation_capabilities classifies table, dict, union, annotated, and string annotations.""" + assert _annotation_capabilities(ann) == expected_caps + + +@pytest.mark.parametrize( + ("ann", "expected"), + [ + pytest.param(TransformContext, True, id="direct"), + pytest.param(_CtxAlias, True, id="type_alias"), + pytest.param(Annotated[TransformContext, "meta"], True, id="annotated"), + pytest.param(Annotated[int, "meta"], False, id="annotated_non_ctx"), + pytest.param(Annotated[int, TransformContext], False, id="annotated_ctx_in_metadata_only"), + pytest.param(TransformContext | None, True, id="union"), + pytest.param(int | None, False, id="union_non_ctx"), + pytest.param(Union.__getitem__((TransformContext, None)), True, id="typing_union_ctx"), + pytest.param(Union.__getitem__((int, str)), False, id="typing_union_non_ctx"), + pytest.param("TransformContext | None", True, id="str_ctx"), + pytest.param("dict[str, Any]", False, id="str_non_ctx"), + pytest.param(int, False, id="non_ctx"), + ], +) +def test_annotation_is_transform_context_detects_variants(ann: object, expected: bool) -> None: + """_annotation_is_transform_context detects TransformContext across aliases, unions, Annotated, and strings.""" + assert _annotation_is_transform_context(ann) is expected + + +@pytest.mark.parametrize( + ("case", "expected"), + [ + pytest.param("kwonly_only", (False, True, False), id="kwonly-only"), + pytest.param("posonly_arrow", (True, False, True), id="posonly-arrow-and-annotated-ctx"), + pytest.param("optional_non_ctx", (False, True, False), id="optional-non-ctx"), + pytest.param("required_second_param", (False, True, True), id="required-second-param"), + pytest.param("str_annotated", (True, False, True), id="string-annotations"), + pytest.param("flagged_attr", (True, True, True), id="accepts-pyarrow-attr"), + ], +) +def test_inspect_transform_signature_handles_parameter_variants( + case: str, + expected: tuple[bool, bool, bool], +) -> None: + """inspect_transform_signature classifies kwonly, posonly, optional, string, and flagged callables.""" + if case == "kwonly_only": + + def fn(*, _flag: bool = False) -> dict[str, Any]: + return {} + + elif case == "posonly_arrow": + + def fn(_table: pa.Table, _info: Annotated[TransformContext, "m"] | None = None, /) -> dict[str, Any]: # type: ignore[misc] + return {} + + elif case == "optional_non_ctx": + + def fn(_row: dict[str, Any], _limit: int = 10) -> dict[str, Any]: # type: ignore[misc] + return {} + + elif case == "required_second_param": + + def fn(_row: dict[str, Any], _extra: int) -> dict[str, Any]: # type: ignore[misc] + return {} + + elif case == "str_annotated": + + def fn(_data: "pa.Table", _info: "TransformContext | None" = None) -> dict[str, Any]: # type: ignore[misc] + return {} + + else: + + def fn(_x: object) -> dict[str, Any]: # type: ignore[misc] + return {} + + fn.accepts_pyarrow = True # type: ignore[attr-defined] + + assert inspect_transform_signature(fn) == expected + + +@pytest.mark.parametrize( + "case", + [ + pytest.param("table_to_dict_with_ctx", id="table-to-dict-with-ctx"), + pytest.param("dict_to_table_with_input_schema", id="dict-to-table-with-input-schema"), + ], +) +def test_invoke_transform_adapts_input_type(case: str) -> None: + """invoke_transform adapts PyArrow tables for dict transforms and dicts for PyArrow transforms.""" + ctx = TransformContext.for_source("items", "input.json") + + if case == "table_to_dict_with_ctx": + + def _dict_with_ctx(row: dict[str, Any], info: TransformContext) -> dict[str, list[dict[str, Any]]]: + return {"items": [{"id": row["id"], "src": info.source_file}]} + + tbl = pa.table({"id": ["1"]}) + assert invoke_transform(_dict_with_ctx, tbl, ctx) == {"items": [{"id": "1", "src": "input.json"}]} + else: + arrow_schema = pa.schema([pa.field("id", pa.string()), pa.field("opt", pa.int64(), nullable=True)]) + + def _arrow_only(table: pa.Table) -> dict[str, pa.Table]: + return {"items": table} + + coerced_out = invoke_transform(_arrow_only, {"id": "1"}, ctx, input_schema=arrow_schema) + assert isinstance(coerced_out["items"], pa.Table) + assert coerced_out["items"].to_pylist() == [{"id": "1", "opt": None}] + + +@pytest.mark.parametrize( + ("initial", "addition", "expect_arrow", "expected_rows"), + [ + pytest.param({}, {"t": [{"a": 1}]}, False, [{"a": 1}], id="new-table-key"), + pytest.param( + {"t": pa.table({"a": [1]})}, + {"t": pa.table({"b": [2]})}, + True, + [{"a": 1, "b": None}, {"a": None, "b": 2}], + id="arrow-plus-arrow-mismatched-columns", + ), + pytest.param( + {"t": [{"a": 1}]}, + {"t": [{"a": 2}]}, + False, + [{"a": 1}, {"a": 2}], + id="list-plus-list", + ), + pytest.param( + {"t": pa.table({"a": [1]})}, + {"t": [{"a": 2}]}, + False, + [{"a": 1}, {"a": 2}], + id="arrow-plus-list", + ), + pytest.param( + {"t": [{"a": 1}]}, + {"t": pa.table({"a": [2]})}, + False, + [{"a": 1}, {"a": 2}], + id="list-plus-arrow", + ), + ], +) +def test_merge_routed_outputs_pass_merges_tables_lists_and_mixed( + initial: dict[str, list[dict[str, Any]] | pa.Table], + addition: dict[str, list[dict[str, Any]] | pa.Table], + *, + expect_arrow: bool, + expected_rows: list[dict[str, Any]], +) -> None: + """merge_routed_outputs merges new keys, mismatched PyArrow tables, row lists, and mixed outputs.""" + merged: dict[str, list[dict[str, Any]] | pa.Table] = dict(initial) + merge_routed_outputs(merged, addition) + + result = merged["t"] + if expect_arrow: + assert isinstance(result, pa.Table) + assert result.to_pylist() == expected_rows + else: + assert isinstance(result, list) + assert result == expected_rows + + +def test_align_and_cast_table_preserves_target_schema_metadata() -> None: + """align_and_cast_table preserves metadata attached to the target PyArrow schema.""" + base_tbl = pa.table({"id": pa.array(["1"], type=pa.string())}) + target_with_meta = pa.schema([pa.field("id", pa.string())], metadata={b"k": b"v"}) + + aligned = align_and_cast_table(base_tbl, target_with_meta) + + assert aligned.schema.metadata == {b"k": b"v"} + + +def test_validate_routed_output_drops_empty_zero_column_table() -> None: + """validate_routed_output drops zero-row, zero-column PyArrow tables.""" + assert validate_routed_output({"empty": pa.table({})}) == {} + + +def test_compose_transforms_fail_empty_list() -> None: + """compose_transforms raises ValueError when given an empty list.""" + with pytest.raises(ValueError, match=r"^transforms must contain at least one callable$"): + compose_transforms([]) + + +def test_route_and_coerce_table_defaults_to_pyarrow_output() -> None: + """route_and_coerce_table returns PyArrow tables by default when use_pyarrow is omitted.""" + base_tbl = pa.table({"id": pa.array(["1"], type=pa.string())}) + + default_routed = route_and_coerce_table(base_tbl, "items") + + assert isinstance(default_routed["items"], pa.Table) + assert default_routed["items"].to_pylist() == [{"id": "1"}] + + +@pytest.mark.parametrize( + ("use_multi_table", "loader_file_format", "expected_keys"), + [ + pytest.param(False, "parquet", {"custom"}, id="no-table-name-parquet"), + pytest.param(False, "jsonl", None, id="no-table-name-jsonl"), + pytest.param(True, "parquet", {"custom", "other"}, id="multi-table-parquet"), + pytest.param(True, "jsonl", None, id="multi-table-jsonl"), + ], +) +def test_transform_settings_mixin_multi_table_and_no_table_name( + monkeypatch: pytest.MonkeyPatch, + use_multi_table: bool, + loader_file_format: str, + expected_keys: set[str] | None, +) -> None: + """TransformSettingsMixin handles multi-table default schema reconciliation and subclasses without table_name.""" + items_schema = ITEMS_SCHEMA + other_schema = pa.schema([pa.field("other_id", pa.string())]) + mod_name = register_temp_module( + monkeypatch, + "_test_settings_mixin_multi_and_none", + TABLE_TRANSFORMS={"items": _identity_items_transform}, + OUTPUT_SCHEMAS={"custom": items_schema}, + ) + + class _NoTableNameSettings(TransformSettingsMixin, BaseModel): + loader_file_format: str = "parquet" + + class _MultiTableSettings(_NoTableNameSettings): + def _resolve_settings_schema_map(self, default_pyarrow_schema: pa.Schema | None = None) -> None: + del default_pyarrow_schema + self._resolve_multi_table_schema_map({"items": items_schema, "other": other_schema}) + + if use_multi_table: + settings: TransformSettingsMixin = _MultiTableSettings( + loader_file_format=loader_file_format, + transforms_module=mod_name, + output_schemas_module=mod_name, + ) + else: + settings = _NoTableNameSettings( + loader_file_format=loader_file_format, + output_schemas_module=mod_name, + ) + + if expected_keys is None: + assert settings.schema_map is None else: - assert result is not None - assert set(result.registry) == expected_tables + expected_registry = {"custom": items_schema} + if "other" in expected_keys: + expected_registry["other"] = other_schema + assert settings.schema_map == ArrowSchemaRegistry(registry=expected_registry) diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py index 17d2da9c..9acc9b75 100644 --- a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py +++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py @@ -1,9 +1,10 @@ """Tests for ncbi_ftp.assembly module — path helpers, file filtering, and FTP assembly downloads.""" from collections.abc import Callable +from contextlib import nullcontext from ftplib import FTP from pathlib import Path, PurePosixPath -from unittest.mock import MagicMock +from unittest.mock import MagicMock, patch import pytest @@ -251,3 +252,59 @@ def fake_retrbinary(_cmd: str, callback: Callable[[bytes], None]) -> None: assert result["files_skipped_checksum_mismatch"] == 1 assert not (dest_dir / "GCF_000001215.4_genomic.fna.gz").exists() assert not (dest_dir / "GCF_000001215.4_genomic.fna.gz.md5").exists() + + +def test_download_assembly_to_local_no_md5_manifest_and_keepalive_args( + tmp_path: Path, +) -> None: + """download_assembly_to_local forwards ftp_host, retries=1, checksum_fn, and keepalive timestamps.""" + mock_ftp = MagicMock(spec=FTP) + assembly_ftp_path = PurePosixPath("/") / "genomes" / "all" / ACC_PATH_215 + keepalive_calls: list[float] = [] + download_calls: list[tuple[PurePosixPath, int, str | None]] = [] + + def fake_keepalive(_ftp: FTP, last_ts: float) -> float: + keepalive_calls.append(last_ts) + return last_ts + 10.0 + + def fake_download( + _ftp: FTP, + remote: PurePosixPath, + local: Path, + *, + retries: int = 3, + checksum_fn: str | None = None, + ) -> str | None: + download_calls.append((remote, retries, checksum_fn)) + local.write_bytes(b"data") + return None + + with ( + patch("cdm_data_loaders.datasets.ncbi_ftp.assembly.ftp_session", return_value=nullcontext(mock_ftp)) as m_sess, + patch( + "cdm_data_loaders.datasets.ncbi_ftp.assembly.ftp_list_dir", + return_value=[ + PurePosixPath("GCF_000001215.4_genomic.fna.gz"), + PurePosixPath("GCF_000001215.4_assembly_stats.txt"), + ], + ), + patch("cdm_data_loaders.datasets.ncbi_ftp.assembly.ftp_noop_keepalive", side_effect=fake_keepalive), + patch("cdm_data_loaders.datasets.ncbi_ftp.assembly.ftp_download_file", side_effect=fake_download), + patch("cdm_data_loaders.datasets.ncbi_ftp.assembly.time.monotonic", side_effect=[100.0, 200.0, 300.0]), + ): + stats = download_assembly_to_local(assembly_ftp_path, tmp_path, ftp_host="ftp.custom.org") + + m_sess.assert_called_once_with("ftp.custom.org", None) + assert stats == { + "accession": "GCF_000001215.4", + "assembly_dir": "GCF_000001215.4_Release_6_plus_ISO1_MT", + "files_downloaded": 2, + "files_skipped_checksum_mismatch": 0, + "files_without_checksum": 2, + } + assert keepalive_calls == [100.0, 200.0] + assert download_calls == [ + (PurePosixPath("GCF_000001215.4_genomic.fna.gz"), 1, None), + (PurePosixPath("GCF_000001215.4_assembly_stats.txt"), 1, None), + ] + assert not (tmp_path / "raw_data" / ACC_PATH_215 / "md5checksums.txt").exists() diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py index babffba2..c1f6bfde 100644 --- a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py +++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py @@ -746,3 +746,104 @@ def test_download_assembly_summary_unknown_database() -> None: """Raise ValueError when database is neither refseq nor genbank.""" with pytest.raises(ValueError, match="Unknown database: bogus"): download_assembly_summary(database="bogus") + + +@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.list_objects") +def test_scan_store_to_synthetic_summary_invokes_progress_callback( + mock_list_objects: MagicMock, +) -> None: + """scan_store_to_synthetic_summary invokes progress_callback with (count, accession).""" + mock_list_objects.return_value = [ + {"Key": "prefix/GCF_000001215.4_Release_6/file.gz", "LastModified": datetime(2024, 1, 1, tzinfo=UTC)}, + ] + progress_calls: list[tuple[int, str]] = [] + + def _record_progress(count: int, acc: str) -> None: + progress_calls.append((count, acc)) + + scan_store_to_synthetic_summary( + PurePosixPath("test-bucket"), + PurePosixPath("prefix"), + "2024/01/31", + progress_callback=_record_progress, + ) + + assert progress_calls == [(1, "GCF_000001215.4")] + + +@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.list_objects") +def test_scan_store_to_synthetic_summary_logs_and_reraises_error( + mock_list_objects: MagicMock, + caplog: pytest.LogCaptureFixture, +) -> None: + """scan_store_to_synthetic_summary logs an error and reraises when listing objects fails.""" + mock_list_objects.side_effect = RuntimeError("s3 down") + + with ( + caplog.at_level("ERROR", logger="cdm_data_loaders.datasets.ncbi_ftp.manifest"), + pytest.raises(RuntimeError, match="s3 down"), + ): + scan_store_to_synthetic_summary(PurePosixPath("test-bucket"), PurePosixPath("prefix"), "2024/01/31") + + assert "Error scanning store" in caplog.text + + +@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.get_existing_object_info") +@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.ftp_retrieve_text") +@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.ftp_noop_keepalive", side_effect=lambda _f, t: t) +@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.connect_ftp") +@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.list_objects") +def test_verify_transfer_candidates_custom_host_progress_and_debug_logs( + mock_list_objects: MagicMock, + mock_connect_ftp: MagicMock, + mock_keepalive: MagicMock, + mock_retrieve: MagicMock, + mock_obj_info: MagicMock, + caplog: pytest.LogCaptureFixture, +) -> None: + """verify_transfer_candidates forwards custom ftp_host, invokes progress_callback, and logs debug and info.""" + custom_host = "ftp.mirror.example.org" + rec_match = AssemblyRecord( + accession="GCF_000001215.4", + status="latest", + seq_rel_date="2024/01/01", + ftp_url=f"https://{custom_host}/genomes/all/GCF/000/001/215/GCF_000001215.4_R6", + assembly_dir=PurePosixPath("GCF_000001215.4_R6"), + ) + rec_mismatch = AssemblyRecord( + accession="GCF_000001405.40", + status="latest", + seq_rel_date="2024/01/01", + ftp_url=f"https://{custom_host}/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38", + assembly_dir=PurePosixPath("GCF_000001405.40_GRCh38"), + ) + mock_list_objects.return_value = [{"Key": "k"}] + mock_connect_ftp.return_value = MagicMock() + digest = "a" * 32 + mock_retrieve.side_effect = [ + f"{digest} ./GCF_000001215.4_R6_genomic.fna.gz\n", + f"{digest} ./GCF_000001405.40_GRCh38_genomic.fna.gz\n", + ] + mock_obj_info.side_effect = [{"Metadata": {"md5": digest}}, None] + + progress: list[tuple[int, int, str]] = [] + + def _record_progress(done: int, total: int, acc: str) -> None: + progress.append((done, total, acc)) + + with caplog.at_level("DEBUG", logger="cdm_data_loaders.datasets.ncbi_ftp.manifest"): + confirmed = verify_transfer_candidates( + ["GCF_000001215.4", "GCF_000001405.40"], + {"GCF_000001215.4": rec_match, "GCF_000001405.40": rec_mismatch}, + PurePosixPath("test-bucket"), + PurePosixPath("prefix"), + ftp_host=custom_host, + progress_callback=_record_progress, + ) + + assert confirmed == ["GCF_000001405.40"] + assert mock_keepalive.called + assert progress == [(1, 2, "GCF_000001215.4"), (2, 2, "GCF_000001405.40")] + assert "Pruned GCF_000001215.4" in caplog.text + assert "File missing from store:" in caplog.text + assert "Checksum verification: 1 confirmed (0 missing from store), 1 pruned (of 2 candidates)" in caplog.text diff --git a/tests/cdm_data_loaders/io/test_clients.py b/tests/cdm_data_loaders/io/test_clients.py index a7039cb0..1648ed0c 100644 --- a/tests/cdm_data_loaders/io/test_clients.py +++ b/tests/cdm_data_loaders/io/test_clients.py @@ -3,6 +3,7 @@ import hashlib import logging from pathlib import Path +from typing import BinaryIO, Self from unittest.mock import AsyncMock, MagicMock, patch import httpx @@ -38,7 +39,8 @@ async def test_download_to_file_system_success( ) -> None: """Simple happy path test of a successful download.""" - def handler(_: httpx.Request) -> httpx.Response: + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" return httpx.Response( 200, content=sample_content, @@ -59,20 +61,48 @@ def handler(_: httpx.Request) -> httpx.Response: @pytest.mark.asyncio -async def test_chunked_streaming(tmp_path: Path, downloader_adapter: DownloaderAdapter) -> None: - """Ensure that chunked downloads work correctly.""" +async def test_chunked_streaming( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, downloader_adapter: DownloaderAdapter +) -> None: + """Ensure that chunked downloads respect the configured chunk size.""" content = b"a" * 10_000 + chunk_size = 1024 + written_sizes: list[int] = [] + real_open = Path.open - def handler(_: httpx.Request) -> httpx.Response: - return httpx.Response(200, content=content) + class TrackingWriter: + def __init__(self, fh: BinaryIO) -> None: + self._fh = fh + + def write(self, data: bytes) -> int: + written_sizes.append(len(data)) + return self._fh.write(data) - downloader = downloader_adapter.make_downloader(handler, chunk_size=1024) + def __enter__(self) -> Self: + self._fh.__enter__() + return self + def __exit__(self, *args: object) -> None: + self._fh.__exit__(*args) + + def handler(request: httpx.Request) -> httpx.Response: + assert request.method == "GET" + return httpx.Response(200, content=content) + + downloader = downloader_adapter.make_downloader(handler, chunk_size=chunk_size) destination = tmp_path / "file.txt" + + def tracking_open(self: Path, mode: str = "r", *args: object, **kwargs: object) -> object: + fh = real_open(self, mode, *args, **kwargs) # type: ignore[call-overload] + if self == destination and mode == "wb": + return TrackingWriter(fh) + return fh + + monkeypatch.setattr(Path, "open", tracking_open) await downloader_adapter.download(downloader, DOWNLOAD_URL, destination) - # TODO: add in count of number of times fh.write is called by _download_once? assert destination.read_bytes() == content + assert written_sizes == [chunk_size] * 9 + [len(content) - chunk_size * 9] @pytest.mark.parametrize( @@ -212,7 +242,7 @@ async def test_checksum_invalid_downloader( @pytest.mark.asyncio async def test_checksum_mismatch(tmp_path: Path, sample_content: bytes, downloader_adapter: DownloaderAdapter) -> None: - """Test checksum checking!""" + """Test checksum mismatch raises ChecksumMismatchError.""" def handler(_: httpx.Request) -> httpx.Response: return httpx.Response(200, content=sample_content) diff --git a/tests/cdm_data_loaders/parsers/test_checkm2.py b/tests/cdm_data_loaders/parsers/test_checkm2.py index 10196f9c..d6d0d62f 100644 --- a/tests/cdm_data_loaders/parsers/test_checkm2.py +++ b/tests/cdm_data_loaders/parsers/test_checkm2.py @@ -63,7 +63,7 @@ def test_get_checkm2_data_headers_only(test_data_dir: Path) -> None: with pytest.raises( RuntimeError, - match=re.escape("no valid data found in checkm2_file"), + match=f"^{re.escape('no valid data found in checkm2_file')}$", ): get_checkm2_data(full_path) @@ -83,3 +83,18 @@ def test_get_checkm2_scores(key: str, test_data_dir: Path) -> None: """Test that checkm2 scores are correctly parsed.""" results_dir = test_data_dir / f"results_{key}" / "quality_report.tsv" assert get_checkm2_data(results_dir) == RESULTS[key]["checkm2"] + + +def test_get_checkm2_data_empty_metric_values(tmp_path: Path) -> None: + """Return None for empty completeness or contamination values.""" + tsv_path = tmp_path / "quality_report.tsv" + tsv_path.write_text( + "Name\tCompleteness\tContamination\nbin_empty_contam\t95.5\t\nbin_empty_comp\t\t1.25\nbin_both_empty\t\t\n", + encoding="utf-8", + ) + + assert get_checkm2_data(tsv_path) == { + "bin_empty_contam": {"checkm2_completeness": 95.5, "checkm2_contamination": None}, + "bin_empty_comp": {"checkm2_completeness": None, "checkm2_contamination": 1.25}, + "bin_both_empty": {"checkm2_completeness": None, "checkm2_contamination": None}, + } diff --git a/tests/cdm_data_loaders/parsers/test_uniprot_metalink.py b/tests/cdm_data_loaders/parsers/test_uniprot_metalink.py index 083330b8..65815e4c 100644 --- a/tests/cdm_data_loaders/parsers/test_uniprot_metalink.py +++ b/tests/cdm_data_loaders/parsers/test_uniprot_metalink.py @@ -1,11 +1,13 @@ """Tests for UniProt metalink file parser.""" import logging +import re from pathlib import Path +from xml.etree.ElementTree import ElementTree import pytest -from cdm_data_loaders.parsers.uniprot_metalink import generate_data_source_table, get_files +from cdm_data_loaders.parsers.uniprot_metalink import generate_data_source_table, get_files, parse_metalink def test_generate_data_source_table(test_data_dir: Path) -> None: @@ -31,17 +33,40 @@ def test_generate_data_source_table(test_data_dir: Path) -> None: ("missing", ["license", "publisher"]), ], ) -def test_generate_data_source_error_missing_fields(test_data_dir: Path, file: str, missing: list[str]) -> None: +def test_generate_data_source_error_missing_fields( + test_data_dir: Path, file: str, missing: list[str], caplog: pytest.LogCaptureFixture +) -> None: """Test the parsing of the data source table.""" metalink_path = test_data_dir / "uniprot" / "metalink" / "invalid" / f"{file}_metalink.xml" missing_fields = ", ".join(missing) + expected_msg = f"Missing required elements from metalink file: {missing_fields}" with pytest.raises( RuntimeError, - match=f"Missing required elements from metalink file: {missing_fields}", + match=f"^{re.escape(expected_msg)}$", ): generate_data_source_table(str(metalink_path)) + assert len(caplog.records) == 1 + assert caplog.records[0].levelno == logging.ERROR + assert caplog.records[0].getMessage() == expected_msg + + +def test_parse_metalink_missing_root( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture +) -> None: + """Log and raise RuntimeError when the parsed XML document has no root element.""" + metalink_path = tmp_path / "empty.xml" + monkeypatch.setattr("cdm_data_loaders.parsers.uniprot_metalink.parse", lambda _path: ElementTree()) + + expected_msg = f"Could not find root in metalink file: {metalink_path}" + with pytest.raises(RuntimeError, match=f"^{re.escape(expected_msg)}$"): + parse_metalink(metalink_path) + + assert len(caplog.records) == 1 + assert caplog.records[0].levelno == logging.ERROR + assert caplog.records[0].getMessage() == expected_msg + UNIPROT_METALINK_DATA = { "uniprot_trembl.fasta.gz": { @@ -152,3 +177,32 @@ def test_get_files( assert caplog.records[0].getMessage() == msg else: assert not caplog.records + + +def test_get_files_without_verification(tmp_path: Path) -> None: + """Return None for checksum fields when a file entry has no verification hash.""" + metalink_path = tmp_path / "no_hash.xml" + metalink_path.write_text( + '\n' + '\n' + " \n" + ' \n' + " 42\n" + " \n" + ' https://example.org/no_hash.txt\n' + " \n" + " \n" + " \n" + "\n", + encoding="utf-8", + ) + + assert get_files(metalink_path) == { + "no_hash.txt": { + "name": "no_hash.txt", + "size": "42", + "checksum": None, + "checksum_fn": None, + "url": "https://example.org/no_hash.txt", + } + } diff --git a/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py b/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py index df566b76..68bb3582 100644 --- a/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py +++ b/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py @@ -627,3 +627,31 @@ def record_download_thread(**_kwargs: Any) -> None: # noqa: ANN401 assert mock_download_client.download.call_count == len(batches) assert main_tid not in download_threads assert len(download_threads) > 1 + + +def test_osf_file_downloader_pass_tags_downloaded_files_table( + test_settings: AtbSettings, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """osf_file_downloader tags yielded items with the downloaded_files table name.""" + monkeypatch.setattr(all_the_bacteria, "download_osf_files", lambda _dir, _files: [{"file_id": "f1"}]) + items = list(osf_file_downloader(test_settings, [])) + assert len(items) == 1 + assert items[0].meta.table_name == "downloaded_files" + + +def test_run_atb_pipeline_pass_passes_dataset_name( + test_settings: AtbSettings, +) -> None: + """run_atb_pipeline forwards test settings and DATASET_NAME to build_file_ingest_kwargs.""" + with ( + patch.object( + all_the_bacteria, + "build_file_ingest_kwargs", + wraps=all_the_bacteria.build_file_ingest_kwargs, + ) as mock_kwargs, + patch.object(all_the_bacteria, "run_pipeline", return_value=None), + ): + run_atb_pipeline(test_settings) + + assert mock_kwargs.call_args.args == (test_settings, DATASET_NAME) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py index 340d917b..82783008 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py @@ -312,3 +312,48 @@ def fail_transform(_row: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: ], "matched_files": ["alpha/part.jsonl", "zeta/part.jsonl"], } + + +def test_build_entity_resource_wires_raw_and_validated_options( + settings_factory: Callable[..., JsonlPydanticIngestSettings], +) -> None: + """build_entity_resource forwards raw resource name, max_table_nesting=0, and parallelized=True.""" + settings = settings_factory() + with ( + patch.object( + jsonlines_base_module, + "build_filesystem_resource", + wraps=jsonlines_base_module.build_filesystem_resource, + ) as mock_fs_resource, + patch.object( + jsonlines_base_module.dlt, + "transformer", + wraps=dlt.transformer, + ) as mock_transformer, + ): + build_entity_resource("widget", settings, _resolve_negative_count_error) + + assert mock_fs_resource.call_args.kwargs["name"] == "widget_raw" + assert mock_fs_resource.call_args.kwargs["max_table_nesting"] == 0 + assert mock_transformer.call_args.kwargs["parallelized"] is True + + +def test_run_registry_ingest_falls_back_when_dataset_name_none( + settings_factory: Callable[..., JsonlPydanticIngestSettings], + widget_model: type[BaseModel], +) -> None: + """run_registry_ingest falls back to default dataset_name when settings.dataset_name is None.""" + settings = settings_factory().model_copy(update={"dataset_name": None}) + mock_load_info = MagicMock(spec=LoadInfo) + with patch.object(jsonlines_base_module, "run_pipeline", return_value=mock_load_info) as mock_run_pipeline: + result = run_registry_ingest( + settings=settings, + registry={"widget": widget_model}, + build_resource=lambda table_name, _entry, cfg: build_entity_resource( + table_name, cfg, _resolve_negative_count_error + ), + pipeline_name="fallback_pipe", + ) + + assert result is mock_load_info + assert mock_run_pipeline.call_args.kwargs["pipeline_kwargs"]["dataset_name"] == "jsonlines_pydantic" diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py index 3b5a4b4f..6beb6d05 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py @@ -288,16 +288,27 @@ def fail_transform(_row: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: } -def test_run_jsonlines_pipeline_uses_subclass_cli_prog_name( +@pytest.mark.parametrize( + ("cli_prog_name", "expected_pipeline_name"), + [ + pytest.param("preset_extract", "preset_extract", id="subclass-cli-prog-name"), + pytest.param(None, EXTRACT_PIPELINE_NAME, id="unset-cli-prog-name-fallback"), + ], +) +def test_run_jsonlines_pipeline_resolves_pipeline_name( extract_settings_factory: Callable[..., JsonlExtractSettings], + cli_prog_name: str | None, + expected_pipeline_name: str, ) -> None: - """run_jsonlines_pipeline forwards a preset subclass cli_prog_name as pipeline_name.""" + """run_jsonlines_pipeline uses subclass cli_prog_name or falls back to EXTRACT_PIPELINE_NAME.""" - class PresetExtractSettings(JsonlExtractSettings, cli_prog_name="preset_extract"): + class CustomExtractSettings(JsonlExtractSettings, cli_prog_name=cli_prog_name): pass base_settings = extract_settings_factory() - preset_settings = PresetExtractSettings.model_validate( + custom_settings = CustomExtractSettings.model_validate( base_settings.model_dump(exclude=set(JsonlExtractSettings.model_computed_fields)) ) - assert_core_run_pipeline_args(extract_pipeline_module, run_jsonlines_pipeline, preset_settings, "preset_extract") + assert_core_run_pipeline_args( + extract_pipeline_module, run_jsonlines_pipeline, custom_settings, expected_pipeline_name + ) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py index dc0c94de..77ee3cbe 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py @@ -1,5 +1,6 @@ """Integration tests for build_entity_resource: file reading, validation, and dlt wiring together.""" +import json from collections.abc import Callable from pathlib import Path from typing import Any @@ -7,12 +8,14 @@ import dlt import pyarrow as pa import pytest -from pydantic import BaseModel +from pydantic import BaseModel, Field, field_validator from cdm_data_loaders.converters.facades.pydantic_to_pyarrow import PydanticToPyArrow from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA from cdm_data_loaders.core.preprocessors import FilePreprocessError from cdm_data_loaders.pipelines.jsonlines.pydantic import ( + PYDANTIC_PIPELINE_NAME, + _make_validator_callbacks, build_entity_resource, run_jsonlines_with_validation_pipeline, ) @@ -426,20 +429,87 @@ def reject_corrupt_file(file_item: Any) -> Any: # noqa: ANN401 assert row["error_detail"] == "corrupt file header" -def test_run_jsonlines_with_validation_pipeline_uses_subclass_cli_prog_name( +@pytest.mark.parametrize( + ("cli_prog_name", "expected_pipeline_name"), + [ + pytest.param("preset_pydantic", "preset_pydantic", id="subclass-cli-prog-name"), + pytest.param(None, PYDANTIC_PIPELINE_NAME, id="default-pipeline-name"), + ], +) +def test_run_jsonlines_with_validation_pipeline_resolves_pipeline_name( scenario_input_dir: Callable[[str], str], settings_factory: Callable[..., JsonlPydanticIngestSettings], tmp_path: Path, + cli_prog_name: str | None, + expected_pipeline_name: str, ) -> None: - """run_jsonlines_with_validation_pipeline uses a preset subclass cli_prog_name as pipeline_name.""" + """run_jsonlines_with_validation_pipeline uses subclass cli_prog_name or defaults to PYDANTIC_PIPELINE_NAME.""" + settings = settings_factory(input_dir=scenario_input_dir("happy_path"), output_dir=str(tmp_path / "out")) + if cli_prog_name is not None: - class PresetPydanticSettings(JsonlPydanticIngestSettings, cli_prog_name="preset_pydantic"): - pass + class PresetPydanticSettings(JsonlPydanticIngestSettings, cli_prog_name=cli_prog_name): + pass - base_settings = settings_factory(input_dir=scenario_input_dir("happy_path"), output_dir=str(tmp_path / "out")) - preset_settings = PresetPydanticSettings.model_validate( - base_settings.model_dump(exclude=set(JsonlPydanticIngestSettings.model_computed_fields)) - ) - load_info = assert_run_ok(run_jsonlines_with_validation_pipeline(preset_settings)) + settings = PresetPydanticSettings.model_validate( + settings.model_dump(exclude=set(JsonlPydanticIngestSettings.model_computed_fields)) + ) + + load_info = assert_run_ok(run_jsonlines_with_validation_pipeline(settings)) + + assert load_info.pipeline.pipeline_name == expected_pipeline_name + + +class AliasWidget(BaseModel): + """Test model with aliased field and custom count validator.""" + + widget_id: str = Field(alias="widgetId") + count: int + + @field_validator("count") + @classmethod + def _must_be_positive(cls, value: int) -> int: + if value < 0: + msg = "count must be non-negative" + raise ValueError(msg) + return value - assert load_info.pipeline.pipeline_name == "preset_pydantic" + +def test_make_validator_callbacks_caches_valid_model_and_serializes_errors() -> None: + """_make_validator_callbacks caches validated models, handles uncached dicts, and serializes non-JSON error contexts.""" + resolve_error, extract = _make_validator_callbacks(AliasWidget) + record: dict[str, Any] = {"widgetId": "w1", "count": 2} + assert resolve_error(record) is None + extracted_model = extract(record) + assert isinstance(extracted_model, AliasWidget) + assert extracted_model.widget_id == "w1" + + # different dict identity triggers fresh validation without relying on stale cache + extracted_uncached = extract({"widgetId": "w2", "count": 3}) + assert isinstance(extracted_uncached, AliasWidget) + assert extracted_uncached.widget_id == "w2" + + # validation error with ValueError in ctx serializes via default=str and clears cached state + assert resolve_error(record) is None + error_json = resolve_error({"widgetId": "w_bad", "count": -1}) + assert error_json is not None + parsed_errors = json.loads(error_json) + assert "count must be non-negative" in parsed_errors[0]["msg"] + + +@pytest.mark.parametrize( + ("extra_kwargs", "record", "expected"), + [ + pytest.param({}, {"widgetId": "w3", "count": 4}, {"widget_id": "w3", "count": 4}, id="default-by-alias-false"), + pytest.param( + {"by_alias": True}, {"widgetId": "w4", "count": 5}, {"widgetId": "w4", "count": 5}, id="by-alias-true" + ), + ], +) +def test_make_validator_callbacks_dump_to_dict_respects_by_alias( + extra_kwargs: dict[str, Any], + record: dict[str, Any], + expected: dict[str, Any], +) -> None: + """_make_validator_callbacks with dump_to_dict=True uses Python field names by default and aliases when by_alias=True.""" + _, extract = _make_validator_callbacks(AliasWidget, dump_to_dict=True, **extra_kwargs) + assert extract(record) == expected diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_routing.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_routing.py index eba9199c..9ed94cf4 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_routing.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_routing.py @@ -410,6 +410,51 @@ def test_make_page_router_pass_buffer_size_controls_batching( {"_rejected": [PREPROCESS_REJECTED_ROW]}, id="preprocessor-rejected-dict", ), + pytest.param( + [ + PREPROCESS_REJECTED_ROW, + make_item( + record={"widget_id": "ok", "count": 1}, raw_record='{"widget_id": "ok", "count": 1}', line_no=1 + ), + ], + { + "widget": [{"widget_id": "ok", "count": 1}], + "_rejected": [PREPROCESS_REJECTED_ROW], + }, + id="pre-rejected-followed-by-valid", + ), + pytest.param( + [ + { + **make_item( + record={"widget_id": "w2", "count": 2}, raw_record='{"widget_id": "w2", "count": 2}', line_no=2 + ), + "error_detail": "ignored", + }, + ], + {"widget": [{"widget_id": "w2", "count": 2}]}, + id="valid-with-extra-error-detail-key", + ), + pytest.param( + [ + { + **make_item(record=None, raw_record="{bad", parse_error="boom", line_no=3), + "error_detail": "unrelated", + }, + ], + { + "_rejected": [ + { + "table_name": "widget", + "source_file": "file.jsonl", + "line_no": 3, + "raw_record": "{bad", + "error_detail": "boom", + } + ], + }, + id="parse-error-precedes-error-detail-key", + ), ], ) def test_make_page_router_pass_use_pyarrow_false_yields_lists_of_dicts( @@ -426,3 +471,24 @@ def test_make_page_router_pass_use_pyarrow_false_yields_lists_of_dicts( assert len(results) == len(expected_pages) assert all(isinstance(page_data, list) for page_data in by_table.values()) assert by_table == expected_pages + + +def test_route_jsonl_lines_coerces_types_via_schema_map( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """route_jsonl_lines passes settings.schema_map to make_record_router so scalar types are coerced before Arrow buffering.""" + directory = tmp_path / "widget" + directory.mkdir() + (directory / "data.jsonl").write_text('{"widget_id": 123}\n', encoding="utf-8") + mod_name = register_temp_module( + monkeypatch, + "_test_route_jsonl_lines_coercion", + OUTPUT_SCHEMAS={"widget": pa.schema([pa.field("widget_id", pa.string())])}, + ) + settings = make_extract_settings(tmp_path, output_schemas_module=mod_name) + + pages = list(route_jsonl_lines(iter(read_items(directory)), settings)) + assert len(pages) == 1 + assert isinstance(pages[0].data, pa.Table) + assert pages[0].data.to_pylist() == [{"widget_id": "123"}] diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_settings.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_settings.py index 811852ea..e9cae5d8 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_settings.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_settings.py @@ -29,6 +29,7 @@ TransformRegistry, ) from cdm_data_loaders.core.settings import CdmDataLoadersBase +from cdm_data_loaders.core.transforms.settings_mixin import TransformSettingsMixin from cdm_data_loaders.datasets.ncbi_datasets.hooks import ( ANNOTATION_REPORT_TABLE, DATASET_REPORT_TABLE, @@ -991,7 +992,17 @@ def test_jsonl_pydantic_settings_fail_invalid_max_recursion_depth( ) -def test_build_output_schema_registry_pass_forwards_max_recursion_depth_and_by_alias() -> None: +@pytest.mark.parametrize( + ("converter_kwargs", "expected_names"), + [ + pytest.param({"max_recursion_depth": 1, "by_alias": True}, ["nodeId", "child"], id="custom-depth-and-alias"), + pytest.param({}, ["node_id", "child"], id="default-options"), + ], +) +def test_build_output_schema_registry_pass_forwards_max_recursion_depth_and_by_alias( + converter_kwargs: dict[str, Any], + expected_names: list[str], +) -> None: """build_output_schema_registry forwards max_recursion_depth and by_alias to PydanticToPyArrow.""" class RecursiveNode(BaseModel): @@ -1001,7 +1012,32 @@ class RecursiveNode(BaseModel): RecursiveNode.model_rebuild() registry = ModelRegistry(registry={"node": RecursiveNode}) - schema_map = build_output_schema_registry(registry, max_recursion_depth=1, by_alias=True) + schema_map = build_output_schema_registry(registry, **converter_kwargs) assert schema_map is not None - expected = PydanticToPyArrow(max_recursion_depth=1, by_alias=True).convert(RecursiveNode) + expected = PydanticToPyArrow(**converter_kwargs).convert(RecursiveNode) assert schema_map.registry["node"].equals(expected, check_metadata=True) + assert schema_map.registry["node"].names == expected_names + + +def test_jsonl_ingest_settings_resolve_entity_registry_schema_map_defaults( + test_defaults: dict[str, Any], +) -> None: + """JsonlIngestSettings._resolve_entity_registry_schema_map defaults max_recursion_depth and by_alias=False.""" + + class PlainTransformSettings(JsonlIngestSettings, TransformSettingsMixin): + def resolve_entity_schema_map(self, registry: ModelRegistry) -> None: + self._resolve_entity_registry_schema_map(registry) + + class AliasedItem(BaseModel): + item_id: str = Field(serialization_alias="itemId") + + (test_defaults["input_dir"] / "node").mkdir(exist_ok=True) + settings = PlainTransformSettings( + **test_defaults, + dataset_name="test_ds", + loader_file_format="parquet", + ) + registry = ModelRegistry(registry={"node": AliasedItem}) + settings.resolve_entity_schema_map(registry) + assert settings.schema_map is not None + assert settings.schema_map.registry["node"].names == ["item_id"] diff --git a/tests/cdm_data_loaders/pipelines/test_runner.py b/tests/cdm_data_loaders/pipelines/test_runner.py index c93c6a2f..a4e0178b 100644 --- a/tests/cdm_data_loaders/pipelines/test_runner.py +++ b/tests/cdm_data_loaders/pipelines/test_runner.py @@ -139,17 +139,23 @@ def test_settings_missing_sync_attrs_pass_fixture_shape( ], ) def test_run_pipeline_fail_invalid_destination_config( - config: dict[str, Any], use_destination: str, tmp_path: Path + config: dict[str, Any], use_destination: str, tmp_path: Path, caplog: pytest.LogCaptureFixture ) -> None: """Invalid destination configs cause the pipeline to fail with an 'Invalid destination config' message.""" settings = CtsSettings(input_dir=tmp_path, output_dir=tmp_path, use_destination=use_destination) # pyright: ignore[reportCallIssue] + expected_msg = f"Invalid destination configuration for {use_destination}" with ( isolated_dlt_config(config), - pytest.raises(ValueError, match=f"Invalid destination configuration for {use_destination}"), + pytest.raises(ValueError, match=expected_msg), ): run_pipeline(settings, []) + error_records = [r for r in caplog.records if r.name == core.__name__ and r.levelno == logging.ERROR] + assert len(error_records) == 1 + assert error_records[0].getMessage() == expected_msg + assert error_records[0].exc_info is not None + @pytest.mark.parametrize( ("config", "use_destination"), @@ -240,12 +246,16 @@ def test_run_pipeline_no_slack_env_var_when_vars_missing( # run_pipeline tests -def test_run_pipeline_minimal(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None: +def test_run_pipeline_minimal( + dlt_test_settings: CtsSettings, destination_bucket: Path, caplog: pytest.LogCaptureFixture +) -> None: """A tiny pipeline loads its records into the destination bucket, not into output_dir.""" + caplog.set_level(logging.INFO) load_info = run_pipeline(dlt_test_settings, tiny_resource()) assert load_info is not None assert not load_info.has_failed_jobs + assert str(load_info) in caplog.messages assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA assert read_destination_rows(Path(dlt_test_settings.output_dir), TINY_TABLE_NAME) == [] diff --git a/tests/cdm_data_loaders/pipelines/test_xml_common.py b/tests/cdm_data_loaders/pipelines/test_xml_common.py new file mode 100644 index 00000000..1f21b141 --- /dev/null +++ b/tests/cdm_data_loaders/pipelines/test_xml_common.py @@ -0,0 +1,132 @@ +"""Unit tests for shared XML pipeline helpers in cdm_data_loaders.pipelines.xml_common.""" + +import logging +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pyarrow as pa +import pytest +from lxml import etree + +from cdm_data_loaders.core.buffer import DictBuffer +from cdm_data_loaders.core.registry import ArrowSchemaRegistry, TransformContext +from cdm_data_loaders.pipelines.xml_common import ( + _log_interval_final, + buffer_single_tag_element, + process_single_tag_xml_file, + stream_and_buffer_element_iter, +) + + +def _convert_id_one(_el: etree._Element) -> dict[str, str]: + return {"id": "1"} + + +def _no_transform(_table: str) -> None: + return None + + +def test_log_interval_final_logs_single_entry_when_not_on_boundary( + caplog: pytest.LogCaptureFixture, + tmp_path: Path, +) -> None: + """Verify _log_interval_final logs when n_entries == 0 and (n_entries + 1) % log_interval != 0.""" + xml_path = tmp_path / "one.xml" + xml_path.write_text("", encoding="utf-8") + with caplog.at_level(logging.DEBUG, logger="cdm_data_loaders.pipelines.xml_common"): + _log_interval_final(0, xml_path, 10) + assert "Processed 1 entries from one.xml" in caplog.text + + +def test_buffer_single_tag_element_uses_buffer_schema_map(tmp_path: Path) -> None: + """Verify buffer_single_tag_element passes buffer.schema_map to route_and_coerce_record.""" + schema = pa.schema([pa.field("id", pa.string()), pa.field("count", pa.int64())]) + buf = DictBuffer(max_items=1, schema_map=ArrowSchemaRegistry(registry={"items": schema})) + el = etree.fromstring(b"a7") + ctx = TransformContext.for_source("items", str(tmp_path / "a.xml")) + + def _convert(_el: etree._Element) -> dict[str, str]: + return {"id": "a", "count": "7"} + + flushed = list( + buffer_single_tag_element( + el, + convert_element=_convert, + transform=None, + buffer=buf, + ctx=ctx, + ) + ) + assert len(flushed) == 1 + table = flushed[0].data + assert isinstance(table, pa.Table) + assert table.schema.equals(schema) + assert table.to_pylist() == [{"id": "a", "count": 7}] + + +def test_process_single_tag_xml_file_missing_target_raises(tmp_path: Path) -> None: + """Verify process_single_tag_xml_file raises RuntimeError when table_name is missing.""" + xml_path = tmp_path / "ns.xml" + xml_path.write_text('1', encoding="utf-8") + missing_settings = SimpleNamespace(table_name=None, xml_tag="item", log_interval=10, buffer_size=10) + + with pytest.raises(RuntimeError, match=r"^Expected table_name and xml_tag when xml_level is not set$"): + list(process_single_tag_xml_file(xml_path, missing_settings, _convert_id_one)) + + +@pytest.mark.parametrize( + ("resolve_default_ns", "expected_rows"), + [ + pytest.param(False, [], id="ignore-default-ns"), + pytest.param(True, [{"id": "1"}], id="resolve-default-ns"), + ], +) +def test_process_single_tag_xml_file_default_namespace( + tmp_path: Path, + resolve_default_ns: bool, + expected_rows: list[dict[str, str]], +) -> None: + """Verify process_single_tag_xml_file matches namespaced tags only when resolve_default_ns is True.""" + xml_path = tmp_path / "ns.xml" + xml_path.write_text('1', encoding="utf-8") + schema = pa.schema([pa.field("id", pa.string())]) + settings = SimpleNamespace( + table_name="items", + xml_tag="item", + log_interval=10, + buffer_size=10, + schema_map=ArrowSchemaRegistry(registry={"items": schema}), + use_pyarrow=True, + get_table_transform=_no_transform, + ) + + pages = list( + process_single_tag_xml_file( + xml_path, + settings, + _convert_id_one, + resolve_default_ns=resolve_default_ns, + ) + ) + + rows = [row for page in pages for row in page.data.to_pylist()] + assert rows == expected_rows + + +def test_stream_and_buffer_element_iter_flushes_and_logs(caplog: pytest.LogCaptureFixture, tmp_path: Path) -> None: + """Verify stream_and_buffer_element_iter iterates elements, logs progress, and flushes buffered items.""" + xml_path = tmp_path / "items.xml" + xml_path.write_text("", encoding="utf-8") + buf = DictBuffer(max_items=10, use_pyarrow=False) + elements = [etree.fromstring(b""), etree.fromstring(b"")] + + def _buffer_fn(_el: etree._Element) -> list[Any]: + return list(buf.add_items({"items": [{"id": "x"}]})) + + with caplog.at_level(logging.DEBUG, logger="cdm_data_loaders.pipelines.xml_common"): + pages = list(stream_and_buffer_element_iter(xml_path, elements, 2, buf, _buffer_fn)) + + assert len(pages) == 1 + assert pages[0].data == [{"id": "x"}, {"id": "x"}] + assert "Processed 2 entries" in caplog.text diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py index 0fb72485..737a2b71 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py @@ -152,16 +152,25 @@ def transform(row: dict) -> dict: ] -def test_run_xmltodict_pipeline_uses_subclass_cli_prog_name( +@pytest.mark.parametrize( + ("cli_prog_name", "expected_pipeline_name"), + [ + pytest.param("preset_xmltodict", "preset_xmltodict", id="subclass-cli-prog-name"), + pytest.param(None, PIPELINE_NAME, id="unset-cli-prog-name-fallback"), + ], +) +def test_run_xmltodict_pipeline_resolves_pipeline_name( settings_factory: Callable[..., XmlToDictSettings], + cli_prog_name: str | None, + expected_pipeline_name: str, ) -> None: - """run_xmltodict_pipeline forwards a preset subclass cli_prog_name as pipeline_name.""" + """run_xmltodict_pipeline uses subclass cli_prog_name or falls back to PIPELINE_NAME.""" - class PresetXmlToDictSettings(XmlToDictSettings, cli_prog_name="preset_xmltodict"): + class CustomXmlToDictSettings(XmlToDictSettings, cli_prog_name=cli_prog_name): pass base_settings = settings_factory() - preset_settings = PresetXmlToDictSettings.model_validate( + custom_settings = CustomXmlToDictSettings.model_validate( base_settings.model_dump(exclude=set(XmlToDictSettings.model_computed_fields)) ) - assert_core_run_pipeline_args(xmltodict_module, run_xmltodict_pipeline, preset_settings, "preset_xmltodict") + assert_core_run_pipeline_args(xmltodict_module, run_xmltodict_pipeline, custom_settings, expected_pipeline_name) diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_processors.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_processors.py index 90e6e6e2..3c547ca4 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_processors.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_processors.py @@ -3,6 +3,7 @@ import gzip import logging from collections.abc import Generator +from datetime import date from pathlib import Path from typing import Final from unittest.mock import MagicMock @@ -26,7 +27,7 @@ ) from cdm_data_loaders.core.buffer import DictBuffer from cdm_data_loaders.core.fields import BUFFER_SIZE, DEFAULTS, LOG_INTERVAL -from cdm_data_loaders.core.registry import ArrowSchemaRegistry +from cdm_data_loaders.core.registry import ArrowSchemaRegistry, TransformContext from cdm_data_loaders.pipelines.xmltopyarrow.processors import process_xml_file_to_pyarrow from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH @@ -613,3 +614,78 @@ def test_process_xml_file_to_pyarrow_pass_use_pyarrow_false_yields_lists_of_dict assert table_name == "people" assert isinstance(data, list) assert data == PEOPLE_XML_TO_PYARROW_PARSED[:2] + + +def test_process_xml_file_to_pyarrow_pass_level_mode_applies_table_transform_and_source_file( + tmp_path: Path, +) -> None: + """Level mode applies settings.get_table_transform(tag_name) and forwards source_file to TransformContext.""" + file_path = write_xml_file(tmp_path, "catalog.xml", MULTI_ENTITY_XML) + settings = fake_xmltopyarrow_level_settings(xml_level=1) + + def transform_book(row: dict[str, object], ctx: TransformContext) -> dict[str, list[dict[str, object]]]: + return { + "book": [{"_id": f"{row['_id']}:{Path(str(ctx.source_file)).name}", "title": str(row["title"]).upper()}], + } + + settings.get_table_transform.side_effect = lambda name: transform_book if name == "book" else None + by_table = dict(table_and_data(process_xml_file_to_pyarrow(file_path, settings))) + + assert by_table["book"].to_pylist() == [ + {"_id": "b1:catalog.xml", "title": "DUNE"}, + {"_id": "b2:catalog.xml", "title": "FOUNDATION"}, + ] + assert by_table["magazine"].to_pylist() == [{"_id": "m1", "issue": 42}] + + +def test_process_xml_file_to_pyarrow_pass_level_mode_coerces_transformed_records_against_schema_map( + tmp_path: Path, +) -> None: + """Level mode forwards buffer.schema_map to route_and_coerce_record so string dates coerce for PyArrow.""" + file_path = write_xml_file(tmp_path, "catalog.xml", MULTI_ENTITY_XML) + settings = fake_xmltopyarrow_level_settings(xml_level=1) + book_dates_schema = pa.schema([pa.field("_id", pa.string()), pa.field("published", pa.date32())]) + assert settings.schema_map is not None + settings.schema_map = ArrowSchemaRegistry( + registry={**settings.schema_map.registry, "book_dates": book_dates_schema} + ) + + def transform_book(row: dict[str, object]) -> dict[str, list[dict[str, object]]]: + return {"book_dates": [{"_id": row["_id"], "published": "2026-01-15"}]} + + settings.get_table_transform.side_effect = lambda name: transform_book if name == "book" else None + by_table = dict(table_and_data(process_xml_file_to_pyarrow(file_path, settings))) + + assert by_table["book_dates"].schema.equals(book_dates_schema, check_metadata=True) + assert by_table["book_dates"].to_pylist() == [ + {"_id": "b1", "published": date(2026, 1, 15)}, + {"_id": "b2", "published": date(2026, 1, 15)}, + ] + + +@pytest.mark.parametrize( + ("missing_attr", "expected_msg"), + [ + pytest.param( + "table_name", "Expected table_name and xml_tag when xml_level is not set", id="missing-table-name" + ), + pytest.param("xml_tag", "Expected table_name and xml_tag when xml_level is not set", id="missing-xml-tag"), + pytest.param( + "xml_element_converter", + "Expected xml_element_converter when xml_level is not set", + id="missing-converter", + ), + ], +) +def test_process_xml_file_to_pyarrow_fail_missing_single_tag_fields( + tmp_path: Path, + missing_attr: str, + expected_msg: str, +) -> None: + """process_xml_file_to_pyarrow raises RuntimeError when single-tag mode is missing required attributes.""" + file_path = write_xml_file(tmp_path, "two.xml", PEOPLE_XML_2) + settings = fake_xmltopyarrow_settings() + setattr(settings, missing_attr, None) + + with pytest.raises(RuntimeError, match=expected_msg): + list(process_xml_file_to_pyarrow(file_path, settings)) diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py index 196c0d13..8cbbbe53 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py @@ -388,3 +388,18 @@ def test_pubmed_cli_pass_file_listing_skips_md5_sidecar_preprocessor( "pubmed25n0001.xml.gz", ], } + + +def test_run_xmltopyarrow_pipeline_falls_back_to_pipeline_name_when_cli_prog_name_unset( + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """run_xmltopyarrow_pipeline falls back to PIPELINE_NAME when cli_prog_name is None.""" + + class NoProgXmlToPyArrowSettings(XmlToPyArrowSettings, cli_prog_name=None): + pass + + base_settings = settings_factory() + no_prog_settings = NoProgXmlToPyArrowSettings.model_validate( + base_settings.model_dump(exclude=set(XmlToPyArrowSettings.model_computed_fields)) + ) + assert_core_run_pipeline_args(xmltopyarrow_module, run_xmltopyarrow_pipeline, no_prog_settings, PIPELINE_NAME) diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py index 301b7ef5..a7295950 100644 --- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py +++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py @@ -8,6 +8,7 @@ import pyarrow as pa import pytest import xmlschema +from lxml import etree from pydantic import ValidationError from pydantic_settings import CliApp, SettingsError @@ -527,3 +528,121 @@ def test_xmltopyarrow_settings_pass_skips_pyarrow_schema_generation_for_jsonl_fo assert settings.pyarrow_schemas == {} assert gen_single_spy.call_count == 0 assert gen_level_spy.call_count == 0 + + +@pytest.mark.parametrize( + ("kwargs", "expected_message"), + [ + pytest.param( + {"xml_level": 1, "xml_tag": "entry"}, + "xml_level cannot be combined with xml_tag or table_name", + id="xml_level_with_xml_tag", + ), + pytest.param( + {"table_name": None, "xml_tag": None}, + "Missing required field(s) when xml_level is not set: table_name, xml_tag", + id="missing_both_table_and_tag", + ), + pytest.param( + {"table_name": "entry", "xml_tag": "entry", "root_tag": "UniRef50"}, + "root_tag can only be specified when xml_level is set", + id="root_tag_without_xml_level", + ), + ], +) +def test_xmltopyarrow_settings_validate_target_selection_exact_messages( + settings_factory: Callable[..., XmlToPyArrowSettings], + kwargs: dict[str, Any], + expected_message: str, +) -> None: + """Target selection validation raises ValidationError with exact error messages.""" + with pytest.raises(ValidationError) as exc_info: + settings_factory(xsd_file=str(PUBMED_XSD_FILE), **kwargs) + assert exc_info.value.errors()[0]["msg"] == f"Value error, {expected_message}" + + +def test_xmltopyarrow_settings_level_mode_with_jsonl_validates_transforms_against_converters( + monkeypatch: pytest.MonkeyPatch, + settings_factory: Callable[..., XmlToPyArrowSettings], +) -> None: + """In level mode with use_pyarrow=False, table validation uses xml_level_converters keys.""" + transforms_mod = register_temp_module( + monkeypatch, + "_test_xmltopyarrow_level_jsonl_transforms", + TABLE_TRANSFORMS={"PubmedArticle": lambda r: {"PubmedArticle": [r]}}, + ) + settings = settings_factory( + xsd_file=str(PUBMED_XSD_FILE), + xml_level=1, + root_tag=PUBMED_ROOT_TAG, + loader_file_format="jsonl", + transforms_module=transforms_mod, + ) + assert settings.pyarrow_schemas == {} + assert "PubmedArticle" in settings.xml_level_converters + + +@pytest.mark.parametrize( + ("depth", "type_predicate", "expected_child_type"), + [ + pytest.param(1, pa.types.is_string, str, id="depth-1-string"), + pytest.param(2, pa.types.is_struct, dict, id="depth-2-struct"), + ], +) +def test_xmltopyarrow_settings_level_mode_forwards_max_recursion_depth( + tmp_path: Path, + settings_factory: Callable[..., XmlToPyArrowSettings], + depth: int, + type_predicate: Callable[[pa.DataType], bool], + expected_child_type: type[object], +) -> None: + """In level mode, max_recursion_depth controls both converter and PyArrow schema recursion depth.""" + recursive_xsd = tmp_path / "recursive.xsd" + recursive_xsd.write_text( + """ + + + + + + + + + + + + + + + +""", + encoding="utf-8", + ) + settings = settings_factory(xsd_file=str(recursive_xsd), xml_level=1, root_tag="root", max_recursion_depth=depth) + + assert type_predicate(settings.pyarrow_schemas["node"].field("child").type) + xml_el = etree.fromstring(b"ab") + row = settings.xml_level_converters["node"](xml_el) + assert isinstance(row["child"], expected_child_type) + + +def test_preset_xmltopyarrow_settings_bind_cdm_transform_passes_default_pyarrow_schema( + settings_factory: Callable[..., XmlToPyArrowSettings], + copy_xsd_fixture: Callable[[str], str], +) -> None: + """_bind_cdm_transform_and_schemas and _resolve_settings_schema_map honor default_pyarrow_schema.""" + custom_schema = pa.schema([pa.field("custom_col", pa.string())]) + + class CustomPresetSettings(PresetXmlToPyArrowSettings, cli_prog_name="custom_preset"): + PRECOMPUTED_SCHEMA: ClassVar[pa.Schema] = pa.schema([pa.field("entry_col", pa.string())]) + + def _resolve_settings_schema_map(self, default_pyarrow_schema: pa.Schema | None = None) -> None: + self._bind_cdm_transform_and_schemas( + lambda r: {"entry": [r]}, + ArrowSchemaRegistry(registry={"entry": custom_schema}), + default_pyarrow_schema=custom_schema if default_pyarrow_schema is None else default_pyarrow_schema, + ) + + xsd_path = copy_xsd_fixture(UNIREF_LIKE_XSD) + settings = settings_factory(CustomPresetSettings, dataset_name="custom_ds", xsd_file=xsd_path) + assert settings.pyarrow_schema is custom_schema diff --git a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py index c0dd96c9..69ae0644 100644 --- a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py @@ -4,6 +4,7 @@ skipped when qsv is not available on PATH or via the QSV_BIN env var. """ +import gzip import sys import threading from collections.abc import Callable @@ -527,3 +528,89 @@ def fail_transform(_row: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: "top.csv", ], } + + +def test_process_xsv_file_pass_gzipped_csv( + settings_factory: Callable[..., XsvIngestSettings], + work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths], +) -> None: + """Valid .csv.gz files are decompressed and loaded into the target table.""" + settings = settings_factory(SIMPLE_SCHEMA, {}) + work_paths = work_paths_factory(settings) + + valid_gz = Path(settings.input_dir) / "good.csv.gz" + valid_gz.write_bytes(gzip.compress(b"number,name\n1,alice\n2,bob\n")) + results = list(_process_xsv_file(make_file_item(valid_gz), settings, work_paths)) + assert len(results) == 1 + assert results[0].meta.table_name == "my_table" + assert results[0].data.to_pylist() == [ + {"number": 1, "name": "alice"}, + {"number": 2, "name": "bob"}, + ] + + +def test_process_xsv_file_fail_corrupt_gzipped_csv_routes_to_rejected( + settings_factory: Callable[..., XsvIngestSettings], + work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths], +) -> None: + """Corrupt .csv.gz files route an error record to _rejected.""" + settings = settings_factory(SIMPLE_SCHEMA, {}) + work_paths = work_paths_factory(settings) + + bad_gz = Path(settings.input_dir) / "corrupt.csv.gz" + bad_gz.write_bytes(b"not-a-valid-gzip-stream") + results = list(_process_xsv_file(make_file_item(bad_gz), settings, work_paths)) + assert [r.meta.table_name for r in results] == [REJECTED_TABLE_NAME] + (rejected,) = results[0].data.to_pylist() + assert rejected["table_name"] == "my_table" + assert rejected["source_file"] == "corrupt.csv.gz" + assert rejected["error_detail"] + + +def test_process_xsv_file_pass_pyarrow_table_transform_routes_table_and_dict_outputs( + settings_factory: Callable[..., XsvIngestSettings], + work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths], + monkeypatch: pytest.MonkeyPatch, +) -> None: + """PyArrow table transforms route pa.Table outputs directly and dict-list outputs through DictBuffer.""" + + def split_table(table: pa.Table) -> dict[str, pa.Table | list[dict[str, Any]]]: + return { + "my_table": table, + "my_table_summary": [{"count": table.num_rows}], + } + + transforms_mod = register_temp_module( + monkeypatch, + "_test_xsv_pyarrow_transforms", + TABLE_TRANSFORMS={"my_table": split_table}, + ) + settings = settings_factory( + SIMPLE_SCHEMA, + {"data.csv": "number,name\n1,alice\n2,bob\n"}, + transforms_module=transforms_mod, + ) + work_paths = work_paths_factory(settings) + + results = list(_process_xsv_file(make_file_item(Path(settings.input_dir) / "data.csv"), settings, work_paths)) + by_table = {r.meta.table_name: r.data for r in results} + + assert set(by_table) == {"my_table", "my_table_summary"} + assert isinstance(by_table["my_table"], pa.Table) + assert by_table["my_table"].to_pylist() == [ + {"number": 1, "name": "alice"}, + {"number": 2, "name": "bob"}, + ] + assert by_table["my_table_summary"] == [{"count": 2}] + + +def test_process_xsv_file_pass_empty_csv_produces_no_rows( + settings_factory: Callable[..., XsvIngestSettings], + work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths], +) -> None: + """A completely empty CSV yields no rows and no errors.""" + settings = settings_factory(SIMPLE_SCHEMA, {"empty.csv": ""}) + work_paths = work_paths_factory(settings) + + results = list(_process_xsv_file(make_file_item(Path(settings.input_dir) / "empty.csv"), settings, work_paths)) + assert results == [] diff --git a/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py b/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py index 85754ced..dead4320 100644 --- a/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py +++ b/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py @@ -5,9 +5,15 @@ from pathlib import Path import pytest -from lxml.etree import XMLSyntaxError, tostring - -from cdm_data_loaders.readers.xml import stream_xml_file, stream_xml_file_at_level +from lxml.etree import XMLSyntaxError, fromstring, tostring + +from cdm_data_loaders.readers.xml import ( + clear_xml_element, + resolve_stream_tag, + stream_elements, + stream_xml_file, + stream_xml_file_at_level, +) from tests.helpers.fs_backends import FsBackend from tests.helpers.xml_samples import SIMPLE_LIBRARY_XML, XML_OPEN_ERROR_CASES, prepare_xml_error_path @@ -336,3 +342,110 @@ def test_stream_xml_file_at_level_fail_negative_level(xml_path_factory: Callable path = xml_path_factory(SIMPLE_LIBRARY_XML) with pytest.raises(ValueError, match="level must be >= 0"): list(stream_xml_file_at_level(path, -1)) + + +def test_clear_xml_element_pass_prunes_preceding_siblings() -> None: + """clear_xml_element clears the target element and removes its preceding siblings.""" + root = fromstring(b"text") + b_elem = root[1] + clear_xml_element(b_elem) + assert [child.tag for child in root] == ["b", "c"] + assert b_elem.text is None + + +def test_clear_xml_element_pass_ignores_non_element() -> None: + """clear_xml_element ignores values that are not lxml elements.""" + clear_xml_element("not-an-element") # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("xml", "filename", "tag", "expected", "expect_unread"), + [ + pytest.param( + NAMESPACED_SIMPLE_XML, + "ns.xml", + f"{{{NAMESPACE_URI}}}book", + f"{{{NAMESPACE_URI}}}book", + True, + id="already-qualified", + ), + pytest.param( + NAMESPACED_SIMPLE_XML, + "ns.xml", + "book", + f"{{{NAMESPACE_URI}}}book", + False, + id="default-namespace-unprefixed", + ), + pytest.param( + SIMPLE_LIBRARY_XML, + "plain.xml", + "book", + "book", + False, + id="no-namespace-unprefixed", + ), + ], +) +def test_resolve_stream_tag_pass_default_namespace( + xml_path_factory: Callable[..., Path], + xml: str, + filename: str, + tag: str, + expected: str, + expect_unread: bool, +) -> None: + """resolve_stream_tag qualifies unprefixed tags from the default namespace without reading pre-qualified tags.""" + path = xml_path_factory(xml, filename=filename) + with path.open("rb") as fh: + assert resolve_stream_tag(fh, tag) == expected + if expect_unread: + assert fh.tell() == 0 + + +@pytest.mark.parametrize( + ("tag", "resolve_default_ns", "expected"), + [ + pytest.param( + "book", + True, + [(f"{{{NAMESPACE_URI}}}book", "1"), (f"{{{NAMESPACE_URI}}}book", "2"), (f"{{{NAMESPACE_URI}}}book", "3")], + id="unprefixed-resolve-enabled", + ), + pytest.param("book", False, [], id="unprefixed-resolve-disabled"), + pytest.param( + f"{{{NAMESPACE_URI}}}book", + True, + [(f"{{{NAMESPACE_URI}}}book", "1"), (f"{{{NAMESPACE_URI}}}book", "2"), (f"{{{NAMESPACE_URI}}}book", "3")], + id="qualified-resolve-enabled", + ), + ], +) +def test_stream_elements_pass_default_namespace_resolution( + xml_path_factory: Callable[..., Path], + tag: str, + resolve_default_ns: bool, + expected: list[tuple[str, str | None]], +) -> None: + """stream_elements resolves unprefixed tags against the default namespace only when resolve_default_ns is True.""" + ns_path = xml_path_factory(NAMESPACED_SIMPLE_XML, filename="ns.xml") + actual = [(el.tag, el.get("id")) for el in stream_elements(ns_path, tag, resolve_default_ns=resolve_default_ns)] + assert actual == expected + + +def test_stream_xml_file_at_level_pass_prunes_intermediate_ancestors( + xml_path_factory: Callable[..., Path], +) -> None: + """stream_xml_file_at_level yields target depth elements while clearing intermediate ancestors.""" + path = xml_path_factory("\n \n 1\n 2\n \n") + elems = list(stream_xml_file_at_level(path, 2)) + assert [el.tag for el in elems] == ["b", "c"] + + +def test_stream_xml_file_at_level_pass_removes_blank_text( + xml_path_factory: Callable[..., Path], +) -> None: + """stream_xml_file_at_level strips indentation whitespace between child elements.""" + path = xml_path_factory(SIMPLE_LIBRARY_XML) + first_book = next(stream_xml_file_at_level(path, 1)) + assert first_book.text is None diff --git a/tests/cdm_data_loaders/readers/xsv/test_duckdb_reader.py b/tests/cdm_data_loaders/readers/xsv/test_duckdb_reader.py index c65d5f4f..c743db27 100644 --- a/tests/cdm_data_loaders/readers/xsv/test_duckdb_reader.py +++ b/tests/cdm_data_loaders/readers/xsv/test_duckdb_reader.py @@ -8,6 +8,7 @@ import pytest from cdm_data_loaders.readers.xsv.duckdb_reader import ( + _read_header_columns, read_validated_rows, resolve_duckdb_column_config, resolve_xsv_parsing_config, @@ -247,3 +248,53 @@ def test_read_validated_rows_pass_emits_pyarrow_tables_without_pa_schema(tmp_pat assert len(tables) == 1 assert isinstance(tables[0], pa.Table) assert tables[0].to_pylist() == [{"a": "1", "b": "foo"}, {"a": "2", "b": "bar"}] + + +def test_read_validated_rows_pass_defaults_to_tab_and_respects_chunk_size(tmp_path: Path) -> None: + """read_validated_rows defaults to tab delimiter and accepts chunk_size in dict mode.""" + pa_schema = pa.schema( + [ + pa.field("a", pa.int64(), nullable=False), + pa.field("b", pa.string(), nullable=False), + ] + ) + xsv_path = write_xsv(tmp_path, "a\tb\n1\tfoo\n2\tbar\n", file_name="data.tsv") + rows = list(read_validated_rows(xsv_path, pa_schema, chunk_size=1)) + assert rows == [{"a": 1, "b": "foo"}, {"a": 2, "b": "bar"}] + + +@pytest.mark.parametrize( + ("content", "file_name", "delimiter", "quote", "escape", "expected"), + [ + pytest.param( + '\n\n"a,1", b \n1,2\n', + "quoted.csv", + ",", + '"', + None, + {"a,1", "b"}, + id="blank-lines-quoted-and-padded", + ), + pytest.param( + '"a\\"1",b\n1,2\n', + "escaped.csv", + ",", + '"', + "\\", + {'a"1', "b"}, + id="escaped-quote", + ), + ], +) +def test_read_header_columns_pass_quoted_escaped_and_padded( + tmp_path: Path, + content: str, + file_name: str, + delimiter: str, + quote: str, + escape: str | None, + expected: set[str], +) -> None: + """_read_header_columns skips blank lines, respects quote and escape, and strips column names.""" + path = write_xsv(tmp_path, content, file_name=file_name) + assert _read_header_columns(path, delimiter, quote, escape) == expected diff --git a/tests/cdm_data_loaders/readers/xsv/test_qsv.py b/tests/cdm_data_loaders/readers/xsv/test_qsv.py index 4e1cc7af..e8110c14 100644 --- a/tests/cdm_data_loaders/readers/xsv/test_qsv.py +++ b/tests/cdm_data_loaders/readers/xsv/test_qsv.py @@ -1966,3 +1966,62 @@ def test_clean_validate_file_pass_end_to_end_dereferences_ref_and_allof_integer_ assert "$defs" not in written_qsv_schema assert written_qsv_schema["properties"]["reads"]["type"] == ["integer", "null"] assert parse_xsv((args.validated_file_dir_path / result).read_text()) == [["gene_id", "reads"], *rows] + + +@pytest.mark.parametrize("use_env", [False, True], ids=["from-path", "from-qsv-bin-env"]) +def test_qsv_check_pass_resolves_binary_from_path_or_env( + monkeypatch: pytest.MonkeyPatch, + qsv_cmd: str, + use_env: bool, +) -> None: + """qsv_check resolves the binary from PATH or falls back to QSV_BIN.""" + if use_env: + monkeypatch.setattr(shutil, "which", lambda _name: None) + monkeypatch.setenv("QSV_BIN", qsv_cmd) + assert qsv_check() == qsv_cmd + + +@pytest.mark.parametrize( + ("qsv_bin", "match"), + [ + pytest.param(None, "Could not locate the qsv binary", id="missing-qsv-bin"), + pytest.param("/nonexistent/qsv-binary", "Cannot perform validation with qsv:", id="nonexistent-qsv-bin"), + ], +) +def test_qsv_check_fail_missing_or_invalid_qsv_bin( + monkeypatch: pytest.MonkeyPatch, + qsv_bin: str | None, + match: str, +) -> None: + """qsv_check raises RuntimeError when the binary is absent or fails to execute.""" + monkeypatch.setattr(shutil, "which", lambda _name: None) + if qsv_bin is None: + monkeypatch.delenv("QSV_BIN", raising=False) + else: + monkeypatch.setenv("QSV_BIN", qsv_bin) + with pytest.raises(RuntimeError, match=match): + qsv_check() + + +def test_run_qsv_validate_pass_preserves_non_xsv_schema_dialect( + make_args: Callable[..., CleanerValidatorArgs], + write_source_file: WriteFile, + write_working_file: WriteFile, +) -> None: + """run_qsv_validate preserves standard $schema URIs when has_xsv_metaschema is False.""" + draft7_uri = "http://json-schema.org/draft-07/schema#" + draft7_schema = ValidatedXsvSchema( + jsonschema={ + "$schema": draft7_uri, + "type": "object", + "required": ["gene_id"], + "properties": {"gene_id": {"type": "string"}}, + } + ) + source = write_source_file("gene_id\nG0001\n", "genes.tsv") + args = make_args(source, first_pass_schema_override=draft7_schema) + input_name = _write_validator_input_file(args, write_working_file, [["G0001"]], "-with-headers") + run_qsv_validate(args, input_name, draft7_schema, first_pass=True) + files = generate_qsv_validate_file_names(args, input_name, first_pass=True) + written = json.loads(files.schema.read_text(encoding="utf-8")) + assert written["$schema"] == draft7_uri diff --git a/tests/cdm_data_loaders/utils/iceberg/test_cleanup.py b/tests/cdm_data_loaders/utils/iceberg/test_cleanup.py index d3d5e35a..e13174ec 100644 --- a/tests/cdm_data_loaders/utils/iceberg/test_cleanup.py +++ b/tests/cdm_data_loaders/utils/iceberg/test_cleanup.py @@ -3,9 +3,11 @@ import gc import importlib import logging +import os import sys import threading import warnings +from pathlib import Path from types import TracebackType from typing import Final from unittest.mock import MagicMock, patch @@ -15,6 +17,7 @@ from dlt.common.libs import pyiceberg as dlt_pyiceberg from dlt.common.libs.pyiceberg import get_iceberg_tables from dlt.destinations.impl.filesystem.filesystem import FilesystemClient +from pyiceberg import catalog as pyiceberg_cat from pyiceberg.catalog import sql as pyiceberg_sql from pyiceberg.catalog.rest import RestCatalog from pyiceberg.catalog.sql import SqlCatalog @@ -28,6 +31,7 @@ from cdm_data_loaders.pipelines.runner import run_pipeline from cdm_data_loaders.utils.iceberg.cleanup import ( DLT_LOAD_CATALOG_ATTR, + PYICEBERG_ENV_CONFIG_ATTR, install_iceberg_catalog_cleanup, ) @@ -229,6 +233,214 @@ def test_iceberg_catalog_cleanup_fail_sql_catalog_integrity_error_becomes_namesp with ( SqlCatalog("dup_ns", uri="sqlite:///:memory:") as catalog, patch("sqlalchemy.orm.Session.commit", side_effect=IntegrityError("INSERT", {}, Exception("UNIQUE"))), - pytest.raises(NamespaceAlreadyExistsError, match="Namespace ns_two already exists"), + pytest.raises(NamespaceAlreadyExistsError, match="Namespace ns_two already exists") as exc_info, ): catalog.create_namespace("ns_two", {"owner": "cdm"}) + assert isinstance(exc_info.value.__cause__, IntegrityError) + + +def _fresh_install_cleanup(monkeypatch: pytest.MonkeyPatch) -> object: + """Reload cleanup module in isolation and install hooks.""" + for target, attr in PATCH_TARGETS: + monkeypatch.setattr(target, attr, getattr(target, attr)) + monkeypatch.delitem(sys.modules, "cdm_data_loaders.utils.iceberg.cleanup", raising=False) + monkeypatch.delattr(iceberg_pkg, "cleanup", raising=False) + fresh_cleanup = importlib.import_module("cdm_data_loaders.utils.iceberg.cleanup") + fresh_cleanup.install_iceberg_catalog_cleanup() + return fresh_cleanup + + +def test_iceberg_catalog_cleanup_pass_rest_catalog_close_invokes_session_close() -> None: + """RestCatalog.close() invokes session.close() via the wrapped close hook.""" + install_iceberg_catalog_cleanup() + mock_session = MagicMock(spec=Session) + with ( + patch.object(RestCatalog, "_fetch_config"), + patch.object(RestCatalog, "_create_session", return_value=mock_session), + ): + catalog = RestCatalog("polaris", uri="http://polaris:8181/api/catalog") + catalog.close() + count_after_first_close = mock_session.close.call_count + assert count_after_first_close > 0 + catalog.close() + assert mock_session.close.call_count == count_after_first_close + 1 + + +def test_iceberg_catalog_cleanup_pass_rest_catalog_without_session_has_no_finalizer() -> None: + """RestCatalog skips registering a finalizer when _create_session returns None.""" + install_iceberg_catalog_cleanup() + with ( + patch.object(RestCatalog, "_fetch_config"), + patch.object(RestCatalog, "_create_session", return_value=None), + ): + no_session_catalog = RestCatalog("polaris_none", uri="http://polaris:8181/api/catalog") + assert not hasattr(no_session_catalog, "_finalizer") + + +@pytest.mark.parametrize( + ("url", "call_kwargs", "expected_kwargs"), + [ + pytest.param("sqlite:///a.db", {}, {"connect_args": {"check_same_thread": False}}, id="sqlite-default"), + pytest.param( + "sqlite:///b.db", + {"connect_args": {"timeout": 10}}, + {"connect_args": {"timeout": 10, "check_same_thread": False}}, + id="sqlite-preserves-other-connect-args", + ), + pytest.param( + "sqlite:///c.db", + {"connect_args": {"check_same_thread": True}}, + {"connect_args": {"check_same_thread": True}}, + id="sqlite-preserves-explicit-check-same-thread", + ), + pytest.param("postgresql://localhost/db", {}, {}, id="non-sqlite-unchanged"), + ], +) +def test_iceberg_catalog_cleanup_pass_create_engine_connect_args( + monkeypatch: pytest.MonkeyPatch, + url: str, + call_kwargs: dict[str, object], + expected_kwargs: dict[str, object], +) -> None: + """Patched create_engine sets check_same_thread=False only for sqlite URLs and preserves existing args.""" + recorded: list[tuple[object, dict[str, object]]] = [] + + def fake_create_engine(engine_url: object, *_args: object, **kwargs: object) -> object: + recorded.append((engine_url, dict(kwargs))) + return object() + + monkeypatch.setattr(pyiceberg_sql, "create_engine", fake_create_engine) + _fresh_install_cleanup(monkeypatch) + + pyiceberg_sql.create_engine(url, **call_kwargs) + assert recorded == [(url, expected_kwargs)] + + +@pytest.mark.parametrize( + ("catalog_name", "create_yaml", "initial_env_home", "expect_tmp_path_home"), + [ + pytest.param("cat1", False, None, False, id="no-yaml-no-env"), + pytest.param("cat2", True, None, True, id="home-yaml-sets-and-restores-env"), + pytest.param("cat3", True, "/custom/home", False, id="existing-env-preserved"), + ], +) +def test_iceberg_catalog_cleanup_pass_load_catalog_env_and_home( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, + catalog_name: str, + create_yaml: bool, + initial_env_home: str | None, + expect_tmp_path_home: bool, +) -> None: + """Patched _load_catalog_from_pyiceberg resets _ENV_CONFIG and temporarily sets PYICEBERG_HOME.""" + seen_env_home: list[str | None] = [] + seen_env_configs: list[object] = [] + + def fake_orig_load(name: str) -> str: + seen_env_home.append(os.environ.get("PYICEBERG_HOME")) + seen_env_configs.append(getattr(pyiceberg_cat, PYICEBERG_ENV_CONFIG_ATTR)) + return f"loaded:{name}" + + monkeypatch.setattr(dlt_pyiceberg, DLT_LOAD_CATALOG_ATTR, fake_orig_load) + _fresh_install_cleanup(monkeypatch) + + monkeypatch.setattr(Path, "home", classmethod(lambda _cls: tmp_path)) + if create_yaml: + (tmp_path / ".pyiceberg.yaml").write_text("catalog: {}\n") + if initial_env_home is None: + monkeypatch.delenv("PYICEBERG_HOME", raising=False) + else: + monkeypatch.setenv("PYICEBERG_HOME", initial_env_home) + + sentinel_cfg = object() + monkeypatch.setattr(pyiceberg_cat, PYICEBERG_ENV_CONFIG_ATTR, sentinel_cfg) + + result = getattr(dlt_pyiceberg, DLT_LOAD_CATALOG_ATTR)(catalog_name) + expected_seen_home = str(tmp_path) if expect_tmp_path_home else initial_env_home + assert result == f"loaded:{catalog_name}" + assert seen_env_home == [expected_seen_home] + assert seen_env_configs[0] is not sentinel_cfg + assert os.environ.get("PYICEBERG_HOME") == initial_env_home + + +def test_iceberg_catalog_cleanup_fail_load_catalog_restores_pyiceberg_home_on_error( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Patched _load_catalog_from_pyiceberg clears temporary PYICEBERG_HOME even when loading raises.""" + + def fake_orig_load(_name: str) -> str: + msg = "load error" + raise ValueError(msg) + + monkeypatch.setattr(dlt_pyiceberg, DLT_LOAD_CATALOG_ATTR, fake_orig_load) + _fresh_install_cleanup(monkeypatch) + monkeypatch.setattr(Path, "home", classmethod(lambda _cls: tmp_path)) + monkeypatch.delenv("PYICEBERG_HOME", raising=False) + (tmp_path / ".pyiceberg.yaml").write_text("catalog: {}\n") + + with pytest.raises(ValueError, match="load error"): + getattr(dlt_pyiceberg, DLT_LOAD_CATALOG_ATTR)("boom") + assert "PYICEBERG_HOME" not in os.environ + + +@pytest.mark.parametrize( + ("catalog", "exc_args"), + [ + pytest.param(None, (None, None, None), id="none-catalog-normal-exit"), + pytest.param(object(), (RuntimeError, RuntimeError("e"), None), id="non-closable-catalog-error-exit"), + ], +) +def test_iceberg_catalog_cleanup_pass_filesystem_client_exit_without_closable_catalog( + monkeypatch: pytest.MonkeyPatch, + catalog: object, + exc_args: tuple[type[BaseException] | None, BaseException | None, TracebackType | None], +) -> None: + """FilesystemClient.__exit__ clears namespaces and delegates to orig __exit__ for None or non-closable catalogs.""" + orig_exits: list[tuple[object, object, object]] = [] + + def fake_orig_exit( + _self: FilesystemClient, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + orig_exits.append((exc_type, exc_val, exc_tb)) + + monkeypatch.setattr(FilesystemClient, "__exit__", fake_orig_exit) + _fresh_install_cleanup(monkeypatch) + + client = object.__new__(FilesystemClient) + client._catalog = catalog + client._catalog_namespaces = {"ns"} + client.__exit__(*exc_args) + assert client._catalog is None + assert client._catalog_namespaces == set() + assert orig_exits == [exc_args] + + +def test_iceberg_catalog_cleanup_fail_filesystem_client_exit_calls_orig_exit_on_close_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """FilesystemClient.__exit__ still calls orig __exit__ when catalog.close() raises.""" + orig_exits: list[tuple[object, object, object]] = [] + + def fake_orig_exit( + _self: FilesystemClient, + exc_type: type[BaseException] | None, + exc_val: BaseException | None, + exc_tb: TracebackType | None, + ) -> None: + orig_exits.append((exc_type, exc_val, exc_tb)) + + monkeypatch.setattr(FilesystemClient, "__exit__", fake_orig_exit) + _fresh_install_cleanup(monkeypatch) + + failing_cat = MagicMock() + failing_cat.close.side_effect = RuntimeError("close failed") + client = object.__new__(FilesystemClient) + client._catalog = failing_cat + client._catalog_namespaces = {"ns"} + with pytest.raises(RuntimeError, match="close failed"): + client.__exit__(None, None, None) + assert orig_exits == [(None, None, None)] diff --git a/tests/cdm_data_loaders/utils/test_arrow_schema.py b/tests/cdm_data_loaders/utils/test_arrow_schema.py index ee4979c2..047d83fc 100644 --- a/tests/cdm_data_loaders/utils/test_arrow_schema.py +++ b/tests/cdm_data_loaders/utils/test_arrow_schema.py @@ -42,7 +42,7 @@ def test_assemble_pyarrow_schema_pass_from_struct_and_sequence( def test_assemble_pyarrow_schema_rejects_empty_component() -> None: """Raise ``ValueError`` when assembling a schema with zero fields.""" - with pytest.raises(ValueError, match="Cannot assemble a PyArrow schema from an empty component"): + with pytest.raises(ValueError, match=r"^Cannot assemble a PyArrow schema from an empty component$"): assemble_pyarrow_schema([]) diff --git a/tests/cdm_data_loaders/utils/test_logging.py b/tests/cdm_data_loaders/utils/test_logging.py index 74656426..7084138d 100644 --- a/tests/cdm_data_loaders/utils/test_logging.py +++ b/tests/cdm_data_loaders/utils/test_logging.py @@ -378,6 +378,23 @@ def test_init_logger_pass_same_mtime_skipped_unless_forced(tmp_path: Path) -> No assert root.handlers[0] not in handlers +def test_init_logger_pass_reapplies_when_switching_config_files_with_same_mtime(tmp_path: Path) -> None: + """Switching to a different config file with the same mtime reapplies logging configuration.""" + config_a = write_json_or_yaml(tmp_path / "logging_a.yaml", make_config("INFO")) + config_b = write_json_or_yaml(tmp_path / "logging_b.yaml", make_config("DEBUG")) + set_mtime(config_b, config_a.stat().st_mtime_ns) + + init_logger(settings_for(config_a)) + root = logging.getLogger() + handlers = list(root.handlers) + assert root.level == logging.INFO + + init_logger(settings_for(config_b)) + assert root.level == logging.DEBUG + assert len(root.handlers) == 1 + assert root.handlers[0] not in handlers + + def test_init_logger_pass_force_relogs_fallback_warning(capsys: pytest.CaptureFixture[str]) -> None: """Forcing the fallback config rebuilds the handler and logs the warning again.""" init_logger(settings_for(None)) diff --git a/tests/cdm_data_loaders/utils/test_progress.py b/tests/cdm_data_loaders/utils/test_progress.py index f71c3eed..b0899c30 100644 --- a/tests/cdm_data_loaders/utils/test_progress.py +++ b/tests/cdm_data_loaders/utils/test_progress.py @@ -292,14 +292,21 @@ def test_make_progress_bar_pass_update_advances_n_regardless_of_disable(disable: def test_make_progress_bar_fail_requires_keyword_arguments() -> None: """All parameters are keyword-only; calling with positional arguments raises TypeError.""" - with pytest.raises(TypeError, match=r"make_progress_bar\(\) takes 0 positional arguments but 2 were given"): + with pytest.raises(TypeError, match=r"make_progress_bar.*? takes 0 positional arguments but 2 were given"): make_progress_bar(100, "test") # type: ignore[misc,call-arg] -@pytest.mark.parametrize("args", [{}, {"total": 10}, {"desc": "test"}]) +@pytest.mark.parametrize( + "args", + [ + pytest.param({}, id="missing-both"), + pytest.param({"total": 10}, id="missing-desc"), + pytest.param({"desc": "test"}, id="missing-total"), + ], +) def test_make_progress_bar_fail_missing_required_arguments(args: dict[str, int | str]) -> None: """Keywords 'total' and 'desc' are required; omitting them raises TypeError.""" - regex = re.compile(r"make_progress_bar\(\) missing [12] required keyword-only arguments?") + regex = re.compile(r"make_progress_bar.*? missing [12] required keyword-only arguments?") with pytest.raises(TypeError, match=regex): make_progress_bar(**args) # type: ignore[call-arg] diff --git a/tests/cdm_data_loaders/validation/jsonschema/test_dereference.py b/tests/cdm_data_loaders/validation/jsonschema/test_dereference.py index df10d589..96730c7e 100644 --- a/tests/cdm_data_loaders/validation/jsonschema/test_dereference.py +++ b/tests/cdm_data_loaders/validation/jsonschema/test_dereference.py @@ -13,6 +13,7 @@ RefSiblingMode, _dereference, _merge_all_of, + _merge_schema_pair, _resolve_ref_and_all_of, _should_apply_ref_siblings, dereference_schema, @@ -874,3 +875,131 @@ def test_dereference_schema_fail_invalid_max_ref_depth(bad_depth: int, exc_cls: """Reject non-positive or boolean max_ref_depth values.""" with pytest.raises(exc_cls, match="max_ref_depth"): dereference_schema({"type": "object"}, max_ref_depth=bad_depth) + + +def _build_ref_chain_schema(hops: int) -> dict[str, Any]: + """Build a Draft 2020-12 schema with a linear $ref chain of `hops` hops.""" + defs: dict[str, Any] = {f"D{i}": {"$ref": f"#/$defs/D{i + 1}"} for i in range(hops - 1)} + defs[f"D{hops - 1}"] = {"type": "string"} + return { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "#/$defs/D0", + "$defs": defs, + } + + +def test_dereference_schema_pass_default_max_ref_depth_allows_fifty_hops() -> None: + """Default max_ref_depth allows a 50-hop $ref chain.""" + assert dereference_schema(_build_ref_chain_schema(50)) == { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "string", + } + + +def test_dereference_schema_fail_default_max_ref_depth_rejects_fifty_one_hops() -> None: + """Default max_ref_depth rejects a 51-hop $ref chain.""" + with pytest.raises(DereferencingError, match=r"exceeded max depth \(50\)"): + dereference_schema(_build_ref_chain_schema(51)) + + +@pytest.mark.parametrize( + ("schema", "expected"), + [ + pytest.param( + {"$ref": "#/$defs/Mid", "title": "Code"}, + {"type": "string", "minLength": 1, "title": "Code"}, + id="chained-ref-with-siblings", + ), + pytest.param( + {"allOf": [{"type": "object", "properties": {"a": {"type": "integer"}}}]}, + {"type": "object", "properties": {"a": {"type": "integer"}}}, + id="allof-resolution", + ), + ], +) +def test_resolve_ref_and_all_of_pass_siblings_chaining_and_allof( + schema: dict[str, Any], + expected: dict[str, Any], +) -> None: + """_resolve_ref_and_all_of resolves chained $ref siblings and allOf and cleans up active state.""" + root = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$defs": { + "Leaf": {"type": "string"}, + "Mid": {"$ref": "#/$defs/Leaf", "minLength": 1}, + }, + } + ctx = _make_ctx(root) + assert _resolve_ref_and_all_of(schema, ctx) == expected + assert ctx.active_refs == set() + assert ctx.active_target_ids == set() + + +@pytest.mark.parametrize( + ("ref_schema", "expected_warning"), + [ + pytest.param( + {"$ref": "#/definitions/Leaf", "title": "T", "description": "D"}, + "Ignoring keyword(s) ['description', 'title'] declared alongside $ref '#/definitions/Leaf'", + id="with-siblings-logs-warning", + ), + pytest.param( + {"$ref": "#/definitions/Leaf"}, + None, + id="without-siblings-no-warning", + ), + ], +) +def test_resolve_ref_and_all_of_pass_draft7_sibling_warning( + caplog: pytest.LogCaptureFixture, + ref_schema: dict[str, Any], + expected_warning: str | None, +) -> None: + """_resolve_ref_and_all_of logs ignored Draft-07 $ref siblings only when siblings are present.""" + draft7_root = { + "$schema": "http://json-schema.org/draft-07/schema#", + "definitions": {"Leaf": {"type": "string"}}, + } + ctx = _make_ctx(draft7_root, spec=DRAFT7) + with caplog.at_level("WARNING"): + result = _resolve_ref_and_all_of(ref_schema, ctx) + assert result == {"type": "string"} + if expected_warning is None: + assert "Ignoring keyword" not in caplog.text + else: + assert expected_warning in caplog.text + + +def test_merge_all_of_fail_non_list_all_of() -> None: + """_merge_all_of raises DereferencingError when allOf is not a list.""" + ctx = _make_ctx({}) + with pytest.raises(DereferencingError, match=r"'allOf' must be a list of schemas, got \{'type': 'string'\}\."): + _merge_all_of({"allOf": {"type": "string"}}, ctx) + + +def test_merge_schema_pair_pass_concatenates_all_of_lists() -> None: + """_merge_schema_pair concatenates allOf lists from both schemas.""" + merged = _merge_schema_pair({"allOf": [{"type": "string"}]}, {"allOf": [{"minLength": 2}]}) + assert merged == {"allOf": [{"type": "string"}, {"minLength": 2}]} + + +def test_dereference_schema_pass_dependencies_and_non_schema_single_keyword() -> None: + """dereference_schema resolves dict and bool dependencies and preserves list dependencies and non-schema values.""" + schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "dependencies": { + "a": ["b"], + "c": {"$ref": "#/definitions/Req"}, + "d": True, + }, + "not": "non-schema-passthrough", + "definitions": {"Req": {"required": ["e"]}}, + } + result = dereference_schema(schema) + assert result["dependencies"] == { + "a": ["b"], + "c": {"required": ["e"]}, + "d": True, + } + assert result["not"] == "non-schema-passthrough" diff --git a/tests/cdm_data_loaders/validation/jsonschema/test_metaschema.py b/tests/cdm_data_loaders/validation/jsonschema/test_metaschema.py index 79abb39b..2e9893f9 100644 --- a/tests/cdm_data_loaders/validation/jsonschema/test_metaschema.py +++ b/tests/cdm_data_loaders/validation/jsonschema/test_metaschema.py @@ -3,7 +3,7 @@ import importlib.resources import json from pathlib import Path -from typing import Any, Final, Never +from typing import Any, Final, Never, Self import httpx import jsonschema.exceptions @@ -486,3 +486,74 @@ def test_check_jsonschema_document_fail_invalid_schemas( """Reject missing, non-string, unsupported, or metaschema-invalid JSON Schema documents.""" with pytest.raises(exc_cls, match=match): check_jsonschema_document(bad_schema) + + +def test_check_jsonschema_document_fail_disallowed_custom_xsv() -> None: + """Reject CUSTOM_META_SCHEMA_URI in check_jsonschema_document when allow_custom_xsv is False.""" + with pytest.raises(ValueError, match=r"Custom metaschema cannot declare itself"): + check_jsonschema_document({"$schema": CUSTOM_META_SCHEMA_URI}, allow_custom_xsv=False) + + +def test_register_xsv_validator_pass_idempotent() -> None: + """register_xsv_validator is idempotent across repeated calls.""" + with preserve_xsv_metaschema_state(): + register_xsv_validator() + first_cls = jsonschema.validators.validator_for({"$schema": CUSTOM_META_SCHEMA_URI}, default=None) # type: ignore[arg-type] + register_xsv_validator() + second_cls = jsonschema.validators.validator_for({"$schema": CUSTOM_META_SCHEMA_URI}, default=None) # type: ignore[arg-type] + assert first_cls is not None + assert first_cls is second_cls + + +def test_register_xsv_validator_pass_adds_missing_id( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """register_xsv_validator injects CUSTOM_META_SCHEMA_URI as $id when missing.""" + no_id_dir = tmp_path / "xsv_config" / "0.1.0" + no_id_dir.mkdir(parents=True) + (no_id_dir / "schema.json").write_text( + json.dumps({"$schema": DRAFT_2020_12_SCHEMA_URI, "type": "object"}), + encoding="utf-8", + ) + with preserve_xsv_metaschema_state(): + monkeypatch.setattr(importlib.resources, "files", lambda _pkg: tmp_path) + register_xsv_validator() + cls = jsonschema.validators.validator_for({"$schema": CUSTOM_META_SCHEMA_URI}, default=None) # type: ignore[arg-type] + assert cls is not None + assert cls.META_SCHEMA["$id"] == CUSTOM_META_SCHEMA_URI + + +def test_load_xsv_meta_schema_fail_forwards_remote_url_and_rejects_self_reference( + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, +) -> None: + """load_xsv_meta_schema forwards remote_url on fallback and rejects schemas declaring remote_url as $schema.""" + custom_url = "https://example.org/custom-xsv/schema.json" + captured_urls: list[str] = [] + + class _FakeResponse: + def raise_for_status(self) -> None: + return None + + def json(self) -> dict[str, Any]: + return {"$schema": custom_url, "type": "object"} + + class _FakeClient: + def __init__(self, **_kwargs: object) -> None: + pass + + def __enter__(self) -> Self: + return self + + def __exit__(self, *_args: object) -> None: + return None + + def get(self, url: str) -> _FakeResponse: + captured_urls.append(url) + return _FakeResponse() + + monkeypatch.setattr(httpx, "Client", _FakeClient) + with pytest.raises(ValueError, match=r"Custom metaschema cannot declare itself"): + load_xsv_meta_schema(local_schema_path=tmp_path / "missing.json", remote_url=custom_url) + assert captured_urls == [custom_url] diff --git a/tests/cdm_data_loaders/validation/xsd/test_particles.py b/tests/cdm_data_loaders/validation/xsd/test_particles.py index 45e88d22..1a48cde9 100644 --- a/tests/cdm_data_loaders/validation/xsd/test_particles.py +++ b/tests/cdm_data_loaders/validation/xsd/test_particles.py @@ -170,3 +170,190 @@ def test_validate_group_particles_rejects_unsupported_constructs(body_xml: str, assert any(is_any_element(item) for item in group.content) with pytest.raises(ValueError, match=expected_match): validate_group_particles(schema, group) + + +GROUP_REF_XSD: Final[str] = """ + + + + + + + + +""" + +NESTED_GROUPS_XSD: Final[str] = """ + + + + + + + + + + + + + + + + + +""" + +GROUP_OCCURS_EDGE_XSD: Final[str] = """ + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + + +class CustomFallbackError(Exception): + """Test-local fallback exception class.""" + + +class CustomCodeError(Exception): + """Test-local code-mapped exception class.""" + + +@pytest.mark.parametrize( + ("xsd_text", "element_tag", "error_by_code", "expected_exc", "expected_match"), + [ + pytest.param( + GROUP_REF_XSD, + "root", + {}, + CustomFallbackError, + "xs:any wildcard element is not supported", + id="group-ref-fallback", + ), + pytest.param( + GROUP_REF_XSD, + "root", + {"E4": CustomCodeError}, + CustomCodeError, + "xs:any wildcard element is not supported", + id="group-ref-mapped-e4", + ), + pytest.param( + NESTED_GROUPS_XSD, + "nestedAbs", + {}, + CustomFallbackError, + "abstract element 'absEl' is not supported", + id="nested-abstract-element-fallback", + ), + pytest.param( + NESTED_GROUPS_XSD, + "nestedAbs", + {"E6": CustomCodeError}, + CustomCodeError, + "abstract element 'absEl' is not supported", + id="nested-abstract-element-mapped-e6", + ), + pytest.param( + NESTED_GROUPS_XSD, + "nestedSub", + {}, + CustomFallbackError, + "substitutionGroup on element 'headEl' is not supported", + id="nested-substitution-group-fallback", + ), + pytest.param( + NESTED_GROUPS_XSD, + "nestedSub", + {"E7": CustomCodeError}, + CustomCodeError, + "substitutionGroup on element 'headEl' is not supported", + id="nested-substitution-group-mapped-e7", + ), + pytest.param( + NESTED_GROUPS_XSD, + "nestedAbsType", + {}, + CustomFallbackError, + "abstract type 'AbsType' is not supported", + id="nested-abstract-type-fallback", + ), + pytest.param( + NESTED_GROUPS_XSD, + "nestedAbsType", + {"E6": CustomCodeError}, + CustomCodeError, + "abstract type 'AbsType' is not supported", + id="nested-abstract-type-mapped-e6", + ), + ], +) +def test_validate_group_particles_fail_group_ref_and_nested_error_forwarding( + xsd_text: str, + element_tag: str, + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], + expected_match: str, +) -> None: + """validate_group_particles resolves xs:group refs, recurses into nested groups, and forwards error classes.""" + schema = xmlschema.XMLSchema(xsd_text) + group = schema.elements[element_tag].type.model_group # type: ignore[union-attr] + assert isinstance(group, XsdGroup) + with pytest.raises(expected_exc, match=expected_match): + validate_group_particles(schema, group, error_type=CustomFallbackError, error_by_code=error_by_code) + + +@pytest.mark.parametrize( + ("element_tag", "expected_min"), + [ + pytest.param("optGroup", 0, id="min-occurs-zero-group"), + pytest.param("emptyGroup", 0, id="empty-group"), + ], +) +def test_group_min_occurs_by_name_pass_zero_and_empty(element_tag: str, expected_min: int) -> None: + """group_min_occurs_by_name returns 0 for minOccurs=0 and empty model groups.""" + schema = xmlschema.XMLSchema(GROUP_OCCURS_EDGE_XSD) + group = schema.elements[element_tag].type.model_group # type: ignore[union-attr] + assert isinstance(group, XsdGroup) + assert group_min_occurs_by_name(group, "a") == expected_min + + +@pytest.mark.parametrize( + ("element_tag", "expected_max"), + [ + pytest.param("zeroMaxGroup", 0, id="max-occurs-zero-group"), + pytest.param("emptyGroup", 0, id="empty-group"), + pytest.param("unboundedGroup", None, id="unbounded-group"), + ], +) +def test_group_max_occurs_by_name_pass_zero_empty_and_unbounded( + element_tag: str, + expected_max: int | None, +) -> None: + """group_max_occurs_by_name handles maxOccurs=0, empty groups, and unbounded groups.""" + schema = xmlschema.XMLSchema(GROUP_OCCURS_EDGE_XSD) + group = schema.elements[element_tag].type.model_group # type: ignore[union-attr] + assert isinstance(group, XsdGroup) + assert group_max_occurs_by_name(group, "a") == expected_max diff --git a/tests/cdm_data_loaders/validation/xsd/test_scalars.py b/tests/cdm_data_loaders/validation/xsd/test_scalars.py index aa8c818c..9f9c599c 100644 --- a/tests/cdm_data_loaders/validation/xsd/test_scalars.py +++ b/tests/cdm_data_loaders/validation/xsd/test_scalars.py @@ -181,3 +181,55 @@ def test_extract_doc_comment_pass_ref_fallback_and_missing( child = children[child_name] fallback_type = None if child_name == "globalRef" else child.type assert extract_doc_comment(child, fallback_type) == expected_doc + + +def test_extract_doc_comment_pass_whitespace_only_without_ref() -> None: + """extract_doc_comment returns None for whitespace-only documentation on a component without ref.""" + no_ref = SimpleNamespace(annotation=SimpleNamespace(documentation=[SimpleNamespace(text=" \n")])) + assert extract_doc_comment(no_ref) is None + + +def test_builtin_xsd_name_pass_missing_attributes() -> None: + """builtin_xsd_name falls back to xs:string when name and base_type attributes are absent.""" + no_name = SimpleNamespace() + assert builtin_xsd_name(no_name) == "xs:string" # type: ignore[arg-type] + + +def _return_false() -> bool: + """Return False for stub XSD type predicates. + + :return: Always ``False``. + :rtype: bool + """ + return False + + +def _return_true() -> bool: + """Return True for stub XSD type predicates. + + :return: Always ``True``. + :rtype: bool + """ + return True + + +def test_resolve_xsd_scalar_kind_pass_unknown_xsd_name() -> None: + """resolve_xsd_scalar_kind falls back to string for an unknown XSD scalar name.""" + unknown_type = SimpleNamespace( + is_union=_return_false, + name="{http://www.w3.org/2001/XMLSchema}unknownCustom", + local_name="unknownCustom", + base_type=None, + ) + assert resolve_xsd_scalar_kind(unknown_type) == ("string", "xs:unknownCustom") # type: ignore[arg-type] + + +def test_iter_simple_builtin_names_pass_list_without_item_type() -> None: + """iter_simple_builtin_names falls back to xs:string for a list type missing item_type.""" + broken_list = SimpleNamespace( + name=None, + base_type=None, + is_list=_return_true, + is_union=_return_false, + ) + assert iter_simple_builtin_names(broken_list) == ["xs:string"] # type: ignore[arg-type] diff --git a/tests/cdm_data_loaders/validation/xsd/test_traversal.py b/tests/cdm_data_loaders/validation/xsd/test_traversal.py index 02c7c11e..de5983f1 100644 --- a/tests/cdm_data_loaders/validation/xsd/test_traversal.py +++ b/tests/cdm_data_loaders/validation/xsd/test_traversal.py @@ -332,3 +332,801 @@ def test_prepare_target_traversal_fail_invalid_max_recursion_depth(bad_depth: in schema = xmlschema.XMLSchema(HIERARCHY_XSD) with pytest.raises(exc_cls, match="max_recursion_depth"): prepare_target_traversal(schema, "record", max_recursion_depth=bad_depth) + + +CAN_REACH_DESCENDANT_XSD: Final[str] = """ + + + + + + + + + + + + + + + + + +""" + +CAN_REACH_VISITED_CYCLE_XSD: Final[str] = """ + + + + + + + + + + + +""" + +FORBIDDEN_ATTR_RECURSION_XSD: Final[str] = """ + + + + +""" + +FORBIDDEN_SIMPLE_CONTENT_RECURSION_XSD: Final[str] = """ + + + + + + + + + +""" + +NAMESPACED_RECURSION_XSD: Final[str] = """ + + + + + + + + + + + + +""" + +DEEP_ELEMENT_XSD: Final[str] = """ + + + + + + + + + + + + + +""" + +AMBIGUOUS_ELEMENT_XSD: Final[str] = """ + + + + + + + + + + + + +""" + +ABSTRACT_ELEMENT_XSD: Final[str] = """ + + + + + + + + + + + + +""" + +NO_GLOBAL_ELEMENTS_XSD: Final[str] = """ + +""" + +MUTUAL_CYCLE_XSD: Final[str] = """ + + + + + + +""" + +SELF_REF_XSD: Final[str] = """ + + + + + + + + + +""" + +ZERO_OCCURS_BEFORE_REF_XSD: Final[str] = """ + + + + + + + + + + +""" + +AMBIGUOUS_ROOTS_XSD: Final[str] = """ + + + + + + +""" + +WILDCARD_ANY_XSD: Final[str] = """ + + + + + + + +""" + +MIXED_NON_STRUCT_PARENTS_XSD: Final[str] = """ + + + + + + + + + + + + +""" + +TARGET_STRUCT_EDGE_XSD: Final[str] = """ + + + + + + + + + + + +""" + +COLLISION_XSD: Final[str] = """ + + + + + + +""" + +SHARED_BAD_DAG_XSD: Final[str] = """ + + + + + + + + + + + +""" + +LEAF_ONLY_XSD: Final[str] = """ + +""" + + +class CustomFallbackError(Exception): + """Test-local fallback exception class.""" + + +class CustomCodeError(Exception): + """Test-local code-mapped exception class.""" + + +@pytest.mark.parametrize( + ("error_by_code", "expected_exc"), + [ + pytest.param({}, CustomFallbackError, id="fallback-error-type"), + pytest.param({"E1": CustomCodeError}, CustomCodeError, id="mapped-error-code"), + ], +) +def test_traversal_context_raise_err_fail_fallback_and_code_map( + hierarchy_schema: xmlschema.XMLSchema, + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], +) -> None: + """TraversalContext.raise_err uses error_by_code when matched and error_type otherwise.""" + ctx = TraversalContext( + schema=hierarchy_schema, + type_counts={}, + max_recursion_depth=2, + error_type=CustomFallbackError, + error_by_code=error_by_code, + ) + with pytest.raises(expected_exc, match="boom"): + ctx.raise_err("E1", "boom") + + +@pytest.mark.parametrize( + ("xsd_text", "expected"), + [ + pytest.param( + """ + + + + """, + True, + id="prohibited-attr-only", + ), + pytest.param( + """ + + + + """, + True, + id="zero-max-occurs-child-only", + ), + pytest.param( + """ + + + + """, + False, + id="optional-attr", + ), + pytest.param( + """ + + + + """, + False, + id="simple-content", + ), + ], +) +def test_is_empty_marker_complex_type_pass_edge_cases(xsd_text: str, expected: bool) -> None: + """is_empty_marker_complex_type handles prohibited attrs, maxOccurs=0, and simpleContent.""" + schema = xmlschema.XMLSchema(xsd_text) + xsd_type = schema.types["T"] + assert isinstance(xsd_type, XsdComplexType) + assert is_empty_marker_complex_type(xsd_type) is expected + + +def test_collect_attributes_pass_skips_prohibited() -> None: + """collect_attributes omits prohibited attributes and keeps optional ones.""" + xsd_ok = """ + + + + + """ + schema = xmlschema.XMLSchema(xsd_ok) + xsd_type = schema.types["T"] + assert isinstance(xsd_type, XsdComplexType) + assert [a.local_name for a in collect_attributes(xsd_type)] == ["keep"] + + +@pytest.mark.parametrize( + ("error_by_code", "expected_exc"), + [ + pytest.param({}, CustomFallbackError, id="fallback-error"), + pytest.param({"E5": CustomCodeError}, CustomCodeError, id="mapped-e5-error"), + ], +) +def test_collect_attributes_fail_wildcard_attribute( + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], +) -> None: + """collect_attributes raises E5 on xs:anyAttribute using error_by_code or error_type.""" + xsd_bad = """ + + + + """ + schema = xmlschema.XMLSchema(xsd_bad) + xsd_type = schema.types["T"] + assert isinstance(xsd_type, XsdComplexType) + with pytest.raises(expected_exc, match="xs:anyAttribute"): + collect_attributes(xsd_type, error_type=CustomFallbackError, error_by_code=error_by_code) + + +@pytest.mark.parametrize( + ("xsd_text", "start_type_name", "target_type_name", "expected"), + [ + pytest.param( + CAN_REACH_DESCENDANT_XSD, + "Wrapper", + "Target", + True, + id="reachable-through-cyclic-middle", + ), + pytest.param( + CAN_REACH_DESCENDANT_XSD, + "ZeroWrapper", + "Target", + False, + id="blocked-by-max-occurs-zero", + ), + pytest.param( + CAN_REACH_VISITED_CYCLE_XSD, + "Root", + "Target", + True, + id="continues-after-visited-cycle", + ), + ], +) +def test_can_reach_any_type_pass_descendants_zero_occurs_and_cycles( + xsd_text: str, + start_type_name: str, + target_type_name: str, + expected: bool, +) -> None: + """can_reach_any_type descends into non-zero complex children and continues past visited cycles.""" + schema = xmlschema.XMLSchema(xsd_text) + start_type = schema.types[start_type_name] + target_type = schema.types[target_type_name] + assert isinstance(start_type, XsdComplexType) + assert can_reach_any_type(start_type, {id(target_type): 1}) is expected + + +@pytest.mark.parametrize( + ("xsd_text", "seed_type_count", "error_type", "error_by_code", "expected_exc", "expected_match"), + [ + pytest.param( + FORBIDDEN_ATTR_RECURSION_XSD, + True, + CustomFallbackError, + {}, + CustomFallbackError, + "@when", + id="date-attribute-fallback", + ), + pytest.param( + FORBIDDEN_SIMPLE_CONTENT_RECURSION_XSD, + True, + CustomFallbackError, + {"E9": CustomCodeError}, + CustomCodeError, + "xs:dateTime", + id="simple-content-datetime-mapped-e9", + ), + pytest.param( + NAMESPACED_RECURSION_XSD, + False, + ValueError, + {}, + ValueError, + "recursive type 'NodeType' contains", + id="namespaced-recursive-type-local-name", + ), + ], +) +def test_check_no_forbidden_types_under_recursion_fail_forbidden_members( + xsd_text: str, + seed_type_count: bool, + error_type: type[Exception], + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], + expected_match: str, +) -> None: + """check_no_forbidden_types_under_recursion rejects date attributes, simpleContent extensions, and namespaced types.""" + schema = xmlschema.XMLSchema(xsd_text) + node_type = schema.types["NodeType"] + type_counts = {id(node_type): 1} if seed_type_count else {} + ctx = TraversalContext( + schema=schema, + type_counts=type_counts, + max_recursion_depth=2, + error_type=error_type, + error_by_code=error_by_code, + ) + with pytest.raises(expected_exc, match=expected_match): + check_no_forbidden_types_under_recursion(node_type, ctx) + + +def test_check_no_forbidden_types_under_recursion_pass_mixed_type() -> None: + """check_no_forbidden_types_under_recursion skips model_group inspection for mixed complex types.""" + schema = xmlschema.XMLSchema(NAMESPACED_RECURSION_XSD) + mixed_type = schema.types["MixedRec"] + ctx = TraversalContext(schema=schema, type_counts={}, max_recursion_depth=2) + check_no_forbidden_types_under_recursion(mixed_type, ctx) + + +def test_resolve_target_element_pass_nested_clark_name() -> None: + """resolve_target_element finds a nested qualified element by Clark-notation tag.""" + schema = xmlschema.XMLSchema(DEEP_ELEMENT_XSD) + found = resolve_target_element(schema, "{http://example.org/ns}item") + assert found.local_name == "item" + + +@pytest.mark.parametrize( + ("xsd_text", "xml_tag", "error_by_code", "expected_exc", "expected_match"), + [ + pytest.param(DEEP_ELEMENT_XSD, "missing", {}, CustomFallbackError, "not found", id="missing-tag-fallback"), + pytest.param( + DEEP_ELEMENT_XSD, + "missing", + {"E1": CustomCodeError}, + CustomCodeError, + "not found", + id="missing-tag-mapped-e1", + ), + pytest.param( + AMBIGUOUS_ELEMENT_XSD, + "dup", + {}, + CustomFallbackError, + "Ambiguous xml_tag 'dup'", + id="ambiguous-tag-fallback", + ), + pytest.param( + AMBIGUOUS_ELEMENT_XSD, + "dup", + {"E2": CustomCodeError}, + CustomCodeError, + "Ambiguous xml_tag 'dup'", + id="ambiguous-tag-mapped-e2", + ), + pytest.param( + ABSTRACT_ELEMENT_XSD, + "absEl", + {}, + CustomFallbackError, + "abstract element 'absEl'", + id="abstract-element-fallback", + ), + pytest.param( + ABSTRACT_ELEMENT_XSD, + "absTyped", + {"E6": CustomCodeError}, + CustomCodeError, + "abstract type on 'absTyped'", + id="abstract-type-mapped-e6", + ), + ], +) +def test_resolve_target_element_fail_missing_ambiguous_and_abstract( + xsd_text: str, + xml_tag: str, + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], + expected_match: str, +) -> None: + """resolve_target_element forwards E1, E2, and E6 errors for missing, ambiguous, and abstract declarations.""" + schema = xmlschema.XMLSchema(xsd_text) + with pytest.raises(expected_exc, match=expected_match): + resolve_target_element(schema, xml_tag, error_type=CustomFallbackError, error_by_code=error_by_code) + + +@pytest.mark.parametrize( + ("error_by_code", "expected_exc"), + [ + pytest.param({}, CustomFallbackError, id="fallback-error"), + pytest.param({"E1": CustomCodeError}, CustomCodeError, id="mapped-e1-error"), + ], +) +def test_resolve_root_elements_fail_no_global_elements( + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], +) -> None: + """resolve_root_elements raises E1 when a schema declares no global elements.""" + schema = xmlschema.XMLSchema(NO_GLOBAL_ELEMENTS_XSD) + with pytest.raises(expected_exc, match="No global elements"): + resolve_root_elements(schema, error_type=CustomFallbackError, error_by_code=error_by_code) + + +@pytest.mark.parametrize( + ("xsd_text", "expected_names"), + [ + pytest.param(MUTUAL_CYCLE_XSD, ["a", "b"], id="mutual-cycle"), + pytest.param(SELF_REF_XSD, ["node", "leaf"], id="self-ref-and-zero-occurs"), + pytest.param(ZERO_OCCURS_BEFORE_REF_XSD, ["skip_target", "root"], id="zero-occurs-before-active-ref"), + ], +) +def test_resolve_root_elements_pass_cycles_and_zero_occurs_refs( + xsd_text: str, + expected_names: list[str], +) -> None: + """resolve_root_elements handles mutual cycles, self-references, and maxOccurs=0 refs.""" + schema = xmlschema.XMLSchema(xsd_text) + assert [e.local_name for e in resolve_root_elements(schema)] == expected_names + + +@pytest.mark.parametrize( + ("xsd_text", "level", "expected_names"), + [ + pytest.param(HIERARCHY_XSD, 0, ["catalog"], id="level-0-root"), + pytest.param( + MIXED_NON_STRUCT_PARENTS_XSD, + 1, + ["k"], + id="skips-simple-mixed-and-duplicate-parents", + ), + ], +) +def test_resolve_elements_at_level_pass_level_zero_and_skips_non_struct_parents( + xsd_text: str, + level: int, + expected_names: list[str], +) -> None: + """resolve_elements_at_level returns root elements at level 0 and skips simple, mixed, and duplicate parents.""" + schema = xmlschema.XMLSchema(xsd_text) + assert list(resolve_elements_at_level(schema, level)) == expected_names + + +@pytest.mark.parametrize( + ("xsd_text", "level", "root_tag", "error_by_code", "expected_exc", "expected_match"), + [ + pytest.param( + HIERARCHY_XSD, + 5, + None, + {}, + CustomFallbackError, + "No XML elements found at level 5", + id="empty-deep-level-fallback", + ), + pytest.param( + HIERARCHY_XSD, + 5, + None, + {"E1": CustomCodeError}, + CustomCodeError, + "No XML elements found at level 5", + id="empty-deep-level-mapped-e1", + ), + pytest.param( + AMBIGUOUS_ROOTS_XSD, + 0, + "missing", + {}, + CustomFallbackError, + "not found", + id="missing-root-tag-fallback", + ), + pytest.param( + AMBIGUOUS_ROOTS_XSD, + 0, + "missing", + {"E1": CustomCodeError}, + CustomCodeError, + "not found", + id="missing-root-tag-mapped-e1", + ), + pytest.param( + AMBIGUOUS_ROOTS_XSD, + 1, + None, + {}, + CustomFallbackError, + "Ambiguous xml_tag 'dup'", + id="ambiguous-child-fallback", + ), + pytest.param( + AMBIGUOUS_ROOTS_XSD, + 1, + None, + {"E2": CustomCodeError}, + CustomCodeError, + "Ambiguous xml_tag 'dup'", + id="ambiguous-child-mapped-e2", + ), + pytest.param( + WILDCARD_ANY_XSD, + 1, + None, + {}, + CustomFallbackError, + "xs:any", + id="wildcard-any-fallback", + ), + pytest.param( + WILDCARD_ANY_XSD, + 1, + None, + {"E4": CustomCodeError}, + CustomCodeError, + "xs:any", + id="wildcard-any-mapped-e4", + ), + ], +) +def test_resolve_elements_at_level_fail_error_forwarding( + xsd_text: str, + level: int, + root_tag: str | None, + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], + expected_match: str, +) -> None: + """resolve_elements_at_level forwards E1, E2, and E4 errors via error_by_code or error_type.""" + schema = xmlschema.XMLSchema(xsd_text) + with pytest.raises(expected_exc, match=expected_match): + resolve_elements_at_level( + schema, + level, + root_tag=root_tag, + error_type=CustomFallbackError, + error_by_code=error_by_code, + ) + + +@pytest.mark.parametrize( + ("tag", "error_by_code", "expected_exc"), + [ + pytest.param("empty", {}, CustomFallbackError, id="empty-fallback"), + pytest.param("empty", {"E3": CustomCodeError}, CustomCodeError, id="empty-mapped-e3"), + pytest.param("plainSc", {}, CustomFallbackError, id="attrless-simple-content-fallback"), + pytest.param("plainSc", {"E3": CustomCodeError}, CustomCodeError, id="attrless-simple-content-mapped-e3"), + ], +) +def test_validate_target_struct_element_fail_empty_and_attrless_simple_content( + tag: str, + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], +) -> None: + """validate_target_struct_element rejects empty and attribute-less simpleContent target elements.""" + schema = xmlschema.XMLSchema(TARGET_STRUCT_EDGE_XSD) + _, elem_type, ctx = prepare_target_traversal( + schema, + tag, + max_recursion_depth=2, + error_type=CustomFallbackError, + error_by_code=error_by_code, + ) + with pytest.raises(expected_exc): + validate_target_struct_element(elem_type, tag, ctx) + + +def test_validate_target_struct_element_pass_simple_content_with_attributes() -> None: + """validate_target_struct_element accepts simpleContent target elements that declare attributes.""" + schema = xmlschema.XMLSchema(TARGET_STRUCT_EDGE_XSD) + _, elem_type, ctx = prepare_target_traversal( + schema, + "attrSc", + max_recursion_depth=2, + error_type=CustomFallbackError, + ) + validate_target_struct_element(elem_type, "attrSc", ctx) + + +@pytest.mark.parametrize( + ("xsd_text", "xml_tag", "error_by_code", "expected_exc", "expected_match"), + [ + pytest.param( + COLLISION_XSD, + "rec", + {"E8": CustomCodeError}, + CustomCodeError, + r"Field name collision.*'_id'", + id="direct-collision-mapped-e8", + ), + pytest.param( + SHARED_BAD_DAG_XSD, + "root", + {}, + CustomFallbackError, + "Field name collision", + id="nested-collision-fallback", + ), + ], +) +def test_validate_complex_subtree_fail_field_name_collision( + xsd_text: str, + xml_tag: str, + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], + expected_match: str, +) -> None: + """validate_complex_subtree detects E8 field-name collisions and forwards error classes.""" + schema = xmlschema.XMLSchema(xsd_text) + _, complex_type, ctx = prepare_target_traversal( + schema, + xml_tag, + max_recursion_depth=2, + error_type=CustomFallbackError, + error_by_code=error_by_code, + ) + with pytest.raises(expected_exc, match=expected_match): + validate_complex_subtree(complex_type, ctx) + + +def test_validate_complex_subtree_pass_skips_visited_type() -> None: + """validate_complex_subtree skips complex types already present in the visited set.""" + schema = xmlschema.XMLSchema(SHARED_BAD_DAG_XSD) + _, root_type, root_ctx = prepare_target_traversal( + schema, + "root", + max_recursion_depth=2, + error_type=CustomFallbackError, + ) + visited = {id(schema.types["SharedBad"])} + validate_complex_subtree(root_type, root_ctx, visited=visited) + + +@pytest.mark.parametrize( + ("xml_tag", "error_by_code", "expected_exc", "expected_match"), + [ + pytest.param("missing", {}, CustomFallbackError, "not found", id="missing-tag-fallback"), + pytest.param( + "leaf", + {"E3": CustomCodeError}, + CustomCodeError, + "must have a complex type", + id="simple-type-mapped-e3", + ), + ], +) +def test_prepare_target_traversal_fail_forwards_error_classes( + xml_tag: str, + error_by_code: dict[str, type[Exception]], + expected_exc: type[Exception], + expected_match: str, +) -> None: + """prepare_target_traversal forwards error_type and error_by_code to element and type resolution.""" + schema = xmlschema.XMLSchema(LEAF_ONLY_XSD) + with pytest.raises(expected_exc, match=expected_match): + prepare_target_traversal( + schema, + xml_tag, + max_recursion_depth=2, + error_type=CustomFallbackError, + error_by_code=error_by_code, + ) diff --git a/uv.lock b/uv.lock index 818f1a8e..0bac33b5 100644 --- a/uv.lock +++ b/uv.lock @@ -34,6 +34,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/bc/3bb8965204ba51bbf4d021bc89d1357cd2179a17ef71b5fc429e21b511a2/aiobotocore-3.10.0-py3-none-any.whl", hash = "sha256:de0ccbcb82b6b0009f1b6667f48442e9d0432f9b8bfe6172fa51777cbea89d03", size = 102759, upload-time = "2026-10-23T10:44:49.109Z" }, ] +[[package]] +name = "aiofile" +version = "3.12.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "caio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/14/31/edb06aabd8f8f0b56d659f30800795f40b93cba96be946ce179f6931e3a5/aiofile-3.12.3.tar.gz", hash = "sha256:caa6aa746b5e47e2165f7abd741b6415e49cf4d44fddc0f61844612cc3924d41", size = 21600, upload-time = "2026-08-04T22:59:27.171Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/79/6e45e778c4c3cab39e0937b007b720c15f76c50c6453d153282d0fcc3588/aiofile-3.12.3-py3-none-any.whl", hash = "sha256:5c1bcc9e929c50834608e8cc1a4cc1d7503eb60c15a535b779fd39e2f372c017", size = 22122, upload-time = "2026-08-04T22:59:25.838Z" }, +] + [[package]] name = "aiohappyeyeballs" version = "2.7.1" @@ -282,6 +294,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, ] +[[package]] +name = "authlib" +version = "1.9.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "joserfc" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/73/69/0215a1818edb1fb6b9dd59b8a3870601f11603b6db1f830ff3f3ea62612f/authlib-1.9.9.tar.gz", hash = "sha256:cca3dffa51c8d80d6df19aee2279e47219dc996f5a527c06ac2e39d0757c99ef", size = 184173, upload-time = "2026-11-17T20:35:21.058Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ba/a5/e79946bc081bfd2e8c0c6125b5a6ae90f8d1541bb224cd7f16ccbb8e71d8/authlib-1.9.9-py2.py3-none-any.whl", hash = "sha256:0199649865fefb4304fc57683eaebb15614523073e41e09d98c72f3268ce5cbc", size = 267584, upload-time = "2026-11-17T20:35:19.249Z" }, +] + [[package]] name = "aws-xray-sdk" version = "2.16.0" @@ -366,6 +391,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/86/3c/7297d87eed9254f6b4823c05b37aa07ec2a99bc5f195760dc574e925eecf/backports_zstd-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:0600e166cb00739a26de74ee1696221a53a4d5dc1f96a0bdeb6b307c1626c15c", size = 322066, upload-time = "2026-10-10T16:36:04.932Z" }, ] +[[package]] +name = "beartype" +version = "0.23.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/79/34/c0cfdaee9c796c8381439fed0fcab62810753b8b0adb1ec88fa7f41eaa5d/beartype-0.23.1.tar.gz", hash = "sha256:8b805f246b32931c74f2b321ea0939fe5bb2ff43c12a3eff19cc4c854b832e1d", size = 1993714, upload-time = "2026-10-11T05:36:55.251Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/cb/2f6aa982e2860c1156830db3da7f823b71784ed7d533ca289ea9b6c6347a/beartype-0.23.1-py3-none-any.whl", hash = "sha256:4461b4dc57e3fdd6c8a8464b22cf118002eea107c33e9fef651c69068ab3cce3", size = 1582101, upload-time = "2026-10-11T05:36:52.431Z" }, +] + [[package]] name = "bioregistry" version = "0.15.12" @@ -480,6 +514,41 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a6/bc/2e35dcb4bfb37a5dd98b8b323158d3103cf263f87164ed03702f6fc5483f/cachetools-8.0.0-py3-none-any.whl", hash = "sha256:5af562e5078f0086771a1e3832a9c3dbf5d336b6e2794a2c12a64680a4c9087d", size = 18610, upload-time = "2026-11-07T07:30:30.831Z" }, ] +[[package]] +name = "caio" +version = "0.12.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/66/a36d1367bd5ef70b6b97b2fb502f682106555e9e49e229f35c0a49d154d5/caio-0.12.10.tar.gz", hash = "sha256:5cecc44c04d5c6291d98b61bdba999f49a7e58dff2d782b4bb0642dc6f7ecf58", size = 84118, upload-time = "2026-10-22T21:35:06.294Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/3e/98ddc781ae6d76407b74f33136f838e444df41a1a43d80373da01da47bbb/caio-0.12.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6b8e0724415ae86889df59af2d44fd6a531851ebc62c9cfdb1cc8235c929524d", size = 49054, upload-time = "2026-10-22T21:34:33.252Z" }, + { url = "https://files.pythonhosted.org/packages/8b/d6/9851c6b24674ef72054c7b05569bd444f1358955c19ba5b5f7e15fa64be6/caio-0.12.10-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:4e72602d4c348d105c9e46dd300b85df3fd3e857e2068370847f5e29879181b5", size = 164861, upload-time = "2026-10-22T21:34:34.472Z" }, + { url = "https://files.pythonhosted.org/packages/85/9a/1f6eac05eb23284c834c08c17cc5f39fafe3754f1d2df94bcf5a2cb93179/caio-0.12.10-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:1d8b6e9275d8f9608b8ecb4b97189e7f1d0b8ed948fb552eb06518d6342a1e53", size = 162592, upload-time = "2026-10-22T21:34:35.566Z" }, + { url = "https://files.pythonhosted.org/packages/1a/be/4bb0538aa4c953c4681c7b309a9cb452eb91640e6b62f250b464f91dfad4/caio-0.12.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cff7a239334b36c4dc82a0ab69aa3244188e5c9e2c30a242d81e4502520f2b1b", size = 162324, upload-time = "2026-10-22T21:34:36.977Z" }, + { url = "https://files.pythonhosted.org/packages/0a/9a/290190821dbc29fccb0ad96e51a7f573394df621da547cb13701de685805/caio-0.12.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a9645aeed1cdeeff34ca78442e6b274c0f801876a6bf0991724d92b7d639655", size = 162171, upload-time = "2026-10-22T21:34:38.173Z" }, + { url = "https://files.pythonhosted.org/packages/39/9d/2a2177a43901f69ef120fb0dfe4a7dc6938e57a086da3ecdeacc12a05635/caio-0.12.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:edda39ac24b4cec74486a06afe7499fc9ddb2c3d5cb98494b2c9fdf95ed7aecd", size = 42798, upload-time = "2026-10-22T21:34:39.444Z" }, + { url = "https://files.pythonhosted.org/packages/b9/0a/5e404a09363c71dfb5f7c20234ee31dc9ad75c7072dc868b6cc5be6f9af3/caio-0.12.10-cp314-cp314-macosx_26_0_arm64.whl", hash = "sha256:3b9c6e6355267f2eccd23e1a17ec2f78dce02ab00b10276999ff7a4615512952", size = 42033, upload-time = "2026-10-22T21:34:40.448Z" }, + { url = "https://files.pythonhosted.org/packages/06/94/41549be5a09a0441f1c1313d5eeb8b60b08034c89cd2c0b5cfeb6a9b9111/caio-0.12.10-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:a2b897245d5e6593a39ea9a52c7b372efc41e5a746c46a2985bdf94622f273fe", size = 165369, upload-time = "2026-10-22T21:34:41.513Z" }, + { url = "https://files.pythonhosted.org/packages/28/47/f272ce0eed9700749934823086434a94d9bea51555724188d8ee660a7fd4/caio-0.12.10-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:40e84b8679f44bee5c2b7cc7093719e15616b6752560809896dc15a8667cd69f", size = 162620, upload-time = "2026-10-22T21:34:42.671Z" }, + { url = "https://files.pythonhosted.org/packages/65/ae/644e7e72a81c4d8ce31dac9865b2a6b38fdc4c56a1418cc2d4d55591be20/caio-0.12.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccd4efddafd48018d30cd560f7c393d177fe01365ee0dcf0bc88528f220732ed", size = 162986, upload-time = "2026-10-22T21:34:43.801Z" }, + { url = "https://files.pythonhosted.org/packages/63/2a/557a90ae71f8411a2cfa0238ad70bddb60178133b7eb41e3fe8e8c854235/caio-0.12.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2138670dd6cab64bd29cfcbd6a7259c0c21646c4dc86a4ec12de907210a2aba9", size = 162258, upload-time = "2026-10-22T21:34:45.271Z" }, + { url = "https://files.pythonhosted.org/packages/78/1a/1ffea8df50383d5bc468e94d2b45d1c458f965b80bb12ad28891d0a55266/caio-0.12.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:79a4e0000dd92b1e4e7842fca99fdaa5fd0259dd33a9892741e7126e74656c49", size = 42801, upload-time = "2026-10-22T21:34:46.485Z" }, + { url = "https://files.pythonhosted.org/packages/f0/59/43da115afd82281312166711ceff5b7ca08d53f79a8955d2e22ac9968eca/caio-0.12.10-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:6fff1f609e54a2e007b6c5c0933f4782ecbf8b6f9144e0930e2dcd2e7440c637", size = 186884, upload-time = "2026-10-22T21:34:47.611Z" }, + { url = "https://files.pythonhosted.org/packages/fa/5d/708aed113a2ae829a265b5ce030bab52ff0e41339cc6047f29bb0b15dfdf/caio-0.12.10-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:925b938bcdf09d1c3df2b6ddd534004b616451ef1810e47f6a63257365376b25", size = 182799, upload-time = "2026-10-22T21:34:48.86Z" }, + { url = "https://files.pythonhosted.org/packages/a1/c3/8de2c1c89548902a389406560e67930333984194217d5bcef6129ff7f18a/caio-0.12.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1ebcac2a14e045aa0a4f63824264a4a845c6b1ac9074d5d673c52ef564a1ada6", size = 185097, upload-time = "2026-10-22T21:34:50.035Z" }, + { url = "https://files.pythonhosted.org/packages/94/43/98997e24fa52f1e6a2a808e446693702773830a863a036a5e727024e7aa0/caio-0.12.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:39e91aab7d0a5f0cac5930562d751e9938ceef1c7ae18477a5689a4e437eada1", size = 182593, upload-time = "2026-10-22T21:34:51.256Z" }, + { url = "https://files.pythonhosted.org/packages/c3/17/78046176bc3572430d03bab1b9e82966bc5b9a28d27c41e6764a20144044/caio-0.12.10-cp315-cp315-macosx_26_0_arm64.whl", hash = "sha256:c0b52bd6fd006d2d07713025f0e1cad61ebd12342f86bfa47e2eff239fbdd01f", size = 42039, upload-time = "2026-10-22T21:34:52.646Z" }, + { url = "https://files.pythonhosted.org/packages/ea/ed/14a90cfe02f14813674a1cc91ff2fad9b0d1bb98260be09b987f579baa28/caio-0.12.10-cp315-cp315-manylinux_2_34_aarch64.whl", hash = "sha256:b7a42ef7379e36e99cc6990aa62465919a3be3fb1f049a40e5aef29fbbd859e1", size = 166510, upload-time = "2026-10-22T21:34:53.703Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b1/9fbeafd6aae00cc0dae96d190cc4fe6e23d20ecd4f26e970878611761fdb/caio-0.12.10-cp315-cp315-manylinux_2_34_x86_64.whl", hash = "sha256:89fe0e3c849a07aba1f76012a50eaf4fb2eaed0398c01bd43912a1bd0f4ab5eb", size = 163501, upload-time = "2026-10-22T21:34:54.988Z" }, + { url = "https://files.pythonhosted.org/packages/34/81/2342f17ab96a7b5d3666cc7dccea59728c4d421e7c954e2937cd817c3be5/caio-0.12.10-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b04f6722884e5fb19690743772875b34306e5ba68d9ecabf7454193bb5caab", size = 164139, upload-time = "2026-10-22T21:34:56.201Z" }, + { url = "https://files.pythonhosted.org/packages/17/b3/e15cfdc902a13d885fa16481c7d01ad04de7b50e0195b9dd5e219ce07614/caio-0.12.10-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a04ac9769fd8eddc1c0ca6308f0f6e668a86cf8d577770c11d85d0aeb584fcf4", size = 163137, upload-time = "2026-10-22T21:34:57.402Z" }, + { url = "https://files.pythonhosted.org/packages/49/ad/7fda5be8557afa8833ea65ebb29feaeeb96cb2b3373fe3220fe1e6899260/caio-0.12.10-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:c8fc80a37f08bba4c2f108277b6eabba564f74963a1a6cab01e50f9ae4434681", size = 42800, upload-time = "2026-10-22T21:34:58.494Z" }, + { url = "https://files.pythonhosted.org/packages/0d/e0/e2692eeb6841614f60ec099eac3c9b10a00dac74adf7321f442a2291ef8d/caio-0.12.10-cp315-cp315t-manylinux_2_34_aarch64.whl", hash = "sha256:3e3799d09b08401e83c30c671f6fa1e1227f6346a3b1336f24cb94c5a8a82b64", size = 187471, upload-time = "2026-10-22T21:34:59.585Z" }, + { url = "https://files.pythonhosted.org/packages/cc/a8/2c3a0cd781732057ab6a8023ef0fe14017b29ef75259295df8f585e3dfba/caio-0.12.10-cp315-cp315t-manylinux_2_34_x86_64.whl", hash = "sha256:ea2e0ea832d36c1e50ab730ba304153f34eb7dc2300b1285fc7cdfae1f1f4075", size = 183526, upload-time = "2026-10-22T21:35:00.764Z" }, + { url = "https://files.pythonhosted.org/packages/62/e2/9f32db16441875afaa2a8a27d9734c2c8421f259163d7d814528160fa6c8/caio-0.12.10-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:8d1071f8cb5d23c8aeccffe36c50a17d32257cddbf913b7e9b6fb524ba42d7a8", size = 185740, upload-time = "2026-10-22T21:35:02.059Z" }, + { url = "https://files.pythonhosted.org/packages/0d/7d/17836b477a1e57a648b90f16a27ebd367d686a02e5b2a4e72b53ae5ad7fe/caio-0.12.10-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:2a54ddadb7db4eab50bfc8b34f36fcf6763e9007f86b92f7cd03f07b542e891e", size = 183396, upload-time = "2026-10-22T21:35:03.492Z" }, + { url = "https://files.pythonhosted.org/packages/5c/c3/1c44005ec4d4c3f7c51b707308efe7836ee16c466f4ff2e2a6f2089eceee/caio-0.12.10-py3-none-any.whl", hash = "sha256:b34a5483b987d207348ebbf86921bc42f08abcde833b91e0caa8760c7daa2f03", size = 26166, upload-time = "2026-10-22T21:35:04.842Z" }, +] + [[package]] name = "cdm-data-loaders" version = "0.1.13" @@ -540,6 +609,7 @@ models = [ ] mutants = [ { name = "mutmut" }, + { name = "mutmut-mcp" }, ] [package.metadata] @@ -595,7 +665,10 @@ models = [ { name = "genson", specifier = ">=1.4.0" }, { name = "json2python-models", specifier = ">=0.3.1" }, ] -mutants = [{ name = "mutmut", specifier = ">=3.8.1" }] +mutants = [ + { name = "mutmut", specifier = ">=3.8.1" }, + { name = "mutmut-mcp", git = "https://github.com/wdm0006/mutmut-mcp" }, +] [[package]] name = "cdm-task-service-client" @@ -1169,6 +1242,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/34/00/5ecde1db0e2bfed9e4613459f07f553145387f20f03fdcb142f9f9c43ccf/curies-0.15.8-py3-none-any.whl", hash = "sha256:f46c58ffedebeecb4a9122eb0df4719907cd0d624be29888a9845f8867887aa2", size = 86104, upload-time = "2026-11-10T19:43:09.901Z" }, ] +[[package]] +name = "cyclopts" +version = "5.5.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "docstring-parser" }, + { name = "rich" }, + { name = "rich-rst" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/5d/cee0096dd8c5ad13ce516599c4823637d0107643666a03ddc43a9580391f/cyclopts-5.5.2.tar.gz", hash = "sha256:c0339b7ddd7b1c886f13bb8b68a6a65aa7144d3027f1d0e290be514b8fe1589b", size = 258336, upload-time = "2026-11-16T16:26:28.322Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7a/b2/be66b75ce0a3ccc7aa3601c9e7f9cd0fbad4301183fc7105c6b9faebeaeb/cyclopts-5.5.2-py3-none-any.whl", hash = "sha256:4fbe863ebace1c65dafc6520c82217915b103d27fbd1d8fcd168f2abcca3a12a", size = 301206, upload-time = "2026-11-16T16:26:26.615Z" }, +] + [[package]] name = "datamodel-code-generator" version = "0.93.0" @@ -1322,6 +1410,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" }, ] +[[package]] +name = "docstring-parser" +version = "0.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" }, +] + [[package]] name = "docutils" version = "0.23" @@ -1384,6 +1481,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" }, ] +[[package]] +name = "exceptiongroup" +version = "1.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/ac/f882fe3234606340c8729372da0fb98c634b97a560c9d375d9afea21496b/exceptiongroup-1.3.2.tar.gz", hash = "sha256:ca82e69f0cb535fbde934885744b67669a3efddd50fe7bc00d3398839fcaddec", size = 34687, upload-time = "2026-11-19T22:10:16.815Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/7d/0fb19ccdf794c448141461180d5992c50402f69f8515a801bfac7ad65f45/exceptiongroup-1.3.2-py3-none-any.whl", hash = "sha256:41c6a70dfec22050d3d8b43386b059a4dc0bca3f3d40a40971cb206fff1785af", size = 16854, upload-time = "2026-11-19T22:10:15.746Z" }, +] + [[package]] name = "executing" version = "2.3.0" @@ -1393,6 +1499,71 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4d/dd/8bc67e7d5ffc1d88aa5af511189dfb76dc4e1e5b808fab186146ef2663e5/executing-2.3.0-py3-none-any.whl", hash = "sha256:736e859c9f8701f11fcf516856f26f562e04776387824b43a35a1dfe21c84122", size = 29393, upload-time = "2026-10-10T14:06:02.777Z" }, ] +[[package]] +name = "fastmcp" +version = "3.4.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "fastmcp-slim", extra = ["client", "server"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f7/77/b5ef8688b0dd3e403b7671106533d62e93252d7d8f3c75f547e4686bba1e/fastmcp-3.4.11.tar.gz", hash = "sha256:36273b04eff98021da4601d4f86c64dfa10e61119e9ddd17ad80650465c65355", size = 28868316, upload-time = "2026-10-20T22:44:36.64Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d5/fb/e6b04a2f4dbce64994f3239267b6899411e8fd9693238c39ee99fe6f2b35/fastmcp-3.4.11-py3-none-any.whl", hash = "sha256:f50a7aa4cc2a5954d02fcc9672ff1d69daff73a02387f959c2647523cd6d5606", size = 8028, upload-time = "2026-10-20T22:44:33.334Z" }, +] + +[[package]] +name = "fastmcp-slim" +version = "3.4.11" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "platformdirs" }, + { name = "pydantic", extra = ["email"] }, + { name = "pydantic-settings" }, + { name = "python-dotenv" }, + { name = "rich" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/35/5f61dc70141df213eddff40bb2b8ccf68b02ecd666da6a1efab7c55b5694/fastmcp_slim-3.4.11.tar.gz", hash = "sha256:50e29b0830fb216aba437f35cfc7dd4bce7451f2f85f8786d6fff3da78d79386", size = 610380, upload-time = "2026-10-20T22:44:12.833Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/71/614669d0a0d8dea87dc0dc684650bb36793027fc409197526532f587fcab/fastmcp_slim-3.4.11-py3-none-any.whl", hash = "sha256:24535dd843b3dcb3272b86c88fdf6ada5026c39ec5b56e2426557223c4d2c8bf", size = 785938, upload-time = "2026-10-20T22:44:10.921Z" }, +] + +[package.optional-dependencies] +client = [ + { name = "authlib" }, + { name = "certifi" }, + { name = "exceptiongroup" }, + { name = "httpx" }, + { name = "mcp" }, + { name = "opentelemetry-api" }, + { name = "py-key-value-aio", extra = ["filetree", "keyring", "memory"] }, + { name = "starlette" }, +] +server = [ + { name = "authlib" }, + { name = "certifi" }, + { name = "cyclopts" }, + { name = "exceptiongroup" }, + { name = "griffelib" }, + { name = "httpx" }, + { name = "joserfc" }, + { name = "jsonref" }, + { name = "jsonschema-path" }, + { name = "mcp" }, + { name = "openapi-pydantic" }, + { name = "opentelemetry-api" }, + { name = "packaging" }, + { name = "py-key-value-aio", extra = ["filetree", "keyring", "memory"] }, + { name = "pyperclip" }, + { name = "python-multipart" }, + { name = "pyyaml" }, + { name = "starlette" }, + { name = "uncalled-for" }, + { name = "uvicorn" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + [[package]] name = "flask" version = "3.1.3" @@ -1698,6 +1869,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] +[[package]] +name = "httpx-sse" +version = "0.4.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, +] + [[package]] name = "humanize" version = "4.16.0" @@ -1933,6 +2113,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, ] +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, +] + +[[package]] +name = "jaraco-context" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, +] + [[package]] name = "jaraco-functools" version = "4.6.0" @@ -1957,6 +2158,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/be/c7/71eb0d51cdcbb63b857c931aa4e9218b078b5d9f8ecd383b234f28fe7b0f/jedi-0.20.1-py2.py3-none-any.whl", hash = "sha256:0fb16d86c4a4c73c37ba518c77419975e30fcc620658a8d14fbb5720cdd34142", size = 4885980, upload-time = "2026-10-12T17:42:21.585Z" }, ] +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + [[package]] name = "jinja2" version = "3.1.6" @@ -2071,6 +2281,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/07/3c953fceee478798f6bdf7b39e0da0793f437e8b5f9e9e1360fba32fb25c/jsonpointer-3.12-py3-none-any.whl", hash = "sha256:89f391335ee2719ae72149e1bf30a4d4c7c5d92f15a3f53faa8d20cc7c6c7f19", size = 10562, upload-time = "2026-11-11T09:34:21.711Z" }, ] +[[package]] +name = "jsonref" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/aa/0d/c1f3277e90ccdb50d33ed5ba1ec5b3f0a242ed8c1b1a85d3afeb68464dca/jsonref-1.1.0.tar.gz", hash = "sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552", size = 8814, upload-time = "2023-01-16T16:10:04.455Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0c/ec/e1db9922bceb168197a558a2b8c03a7963f1afe93517ddd3cf99f202f996/jsonref-1.1.0-py3-none-any.whl", hash = "sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9", size = 9425, upload-time = "2023-01-16T16:10:02.255Z" }, +] + [[package]] name = "jsonschema" version = "4.26.0" @@ -2155,6 +2374,23 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl", hash = "sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407", size = 29032, upload-time = "2025-10-16T19:19:16.783Z" }, ] +[[package]] +name = "keyring" +version = "25.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "secretstorage", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, +] + [[package]] name = "lazy-object-proxy" version = "1.13.0" @@ -2564,6 +2800,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" }, ] +[[package]] +name = "mcp" +version = "1.30.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "httpx" }, + { name = "httpx-sse" }, + { name = "jsonschema" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyjwt", extra = ["crypto"] }, + { name = "python-multipart" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "sse-starlette" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, + { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ba/93/0142dc84a666daf8ad51a34268f34c12fd6fda4f3810c4be2504eecc8212/mcp-1.30.0.tar.gz", hash = "sha256:445414625fce5c295faa505bb11bacece661ab6f4028d57c935db57820b7a3e4", size = 680511, upload-time = "2026-09-07T14:34:15.845Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/f4/e58bc33317c92a0203664daaf00bf6f41166cc0149e5d6870a03f7cd004a/mcp-1.30.0-py3-none-any.whl", hash = "sha256:666edb5009503e1047c9d60346a756f94b261f05cc2625f23d41c728ffc484d0", size = 234581, upload-time = "2026-09-07T14:34:14.266Z" }, +] + [[package]] name = "mdit-py-plugins" version = "0.7.0" @@ -2999,6 +3260,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/0d/1885a8307b845e5b44ddf568939900819653e0168356ab94ed670b0e7ed0/mutmut-3.8.1-py3-none-any.whl", hash = "sha256:e424d72c0557b6a52a721d9b71e2d18eb9590af746db0bbad50b71e8d4d8ed78", size = 82376, upload-time = "2026-10-28T07:54:36.982Z" }, ] +[[package]] +name = "mutmut-mcp" +version = "0.1.0" +source = { git = "https://github.com/wdm0006/mutmut-mcp#7aeef920d3bcc05ce99a829ed8de9dcfb283faf3" } +dependencies = [ + { name = "fastmcp" }, + { name = "mutmut" }, +] + [[package]] name = "mypy-extensions" version = "1.2.0" @@ -3088,6 +3358,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/7f/c2d1b436b6e7cfebac140c2579a298344b85f2991a2ce5c3615cefb29400/numpy-2.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:7a14a461d9340f1b46b8648578aed9cdb8b3b018a8fac6c1dde2c9192a01a87f", size = 10883718, upload-time = "2026-10-10T20:05:28.547Z" }, ] +[[package]] +name = "openapi-pydantic" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/32/0c9bd3e4e847cd6117b64dbef4cf810faa9cdbd6689323b811569cc8a1b8/openapi_pydantic-0.6.0.tar.gz", hash = "sha256:11f3ac6ad41521fc156381ec587246b0c0ecea523bd9565ed74d3e7fac9d91cd", size = 1667049, upload-time = "2026-10-02T12:20:00.508Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/31/57/9d3eb823a03c33e9d4c810e6a3276e3a6dc820a7fdc0961db2c04e613817/openapi_pydantic-0.6.0-py3-none-any.whl", hash = "sha256:8bd6d548a250da37db96f461bcd52b3468e8834ad7d6db14b00a6b2ebbd163a3", size = 96931, upload-time = "2026-10-02T12:19:58.917Z" }, +] + [[package]] name = "openapi-schema-validator" version = "0.9.0" @@ -3134,6 +3416,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" }, ] +[[package]] +name = "opentelemetry-api" +version = "1.45.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/14/cf3a6b6448849656e1e5f4e2101f3988dc6789724050096aba1bc5ccf592/opentelemetry_api-1.45.2.tar.gz", hash = "sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4", size = 73277, upload-time = "2026-10-29T17:29:37.888Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a9/2d/610db864d7513158cebb8f36cc88e67e73bd6c7aa1905ef18d89e1281c81/opentelemetry_api-1.45.2-py3-none-any.whl", hash = "sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34", size = 60335, upload-time = "2026-10-29T17:29:06.955Z" }, +] + [[package]] name = "ordered-set" version = "4.1.0" @@ -3566,6 +3860,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/18/83376915176eb058cb86470eb7396388a13350b09a8f233a79303bbcbc5b/pure_eval-0.2.4-py3-none-any.whl", hash = "sha256:96cae060a313cfaad51bb761278bfb0e62dc0248d9315a81173752dc546cd37a", size = 11893, upload-time = "2026-09-10T21:41:21.532Z" }, ] +[[package]] +name = "py-key-value-aio" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "beartype" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ca/99/c346e3474853801ec5ecf4c3ee60cfc6060327ebc31424df4e346620dde4/py_key_value_aio-0.4.6.tar.gz", hash = "sha256:267c03c3e24cb99d3097612f8a5cfd8e11785c6a2975e272db0e303ea1850bfd", size = 112876, upload-time = "2026-09-18T15:29:52.002Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9a/46/12871550fb0613d149144bb9d41f22680d5f46717bc45a76eb5798c20c3f/py_key_value_aio-0.4.6-py3-none-any.whl", hash = "sha256:820b30c7959fc738ea3d00160cdb10cd258f51e38737b2add530defeea1ded77", size = 171289, upload-time = "2026-09-18T15:29:50.665Z" }, +] + +[package.optional-dependencies] +filetree = [ + { name = "aiofile" }, + { name = "anyio" }, +] +keyring = [ + { name = "keyring" }, +] +memory = [ + { name = "cachetools" }, +] + [[package]] name = "py-partiql-parser" version = "0.6.3" @@ -3847,6 +4166,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1b/ac/e5a5c90e26edc068408281c6c2c1c8fbb6aae6efd1b312cb2cdd629726c0/pyjsg-0.12.5-py3-none-any.whl", hash = "sha256:6cf60375f203dd275db24fb73b0874b6e1f7a90ef3eaa1e4dcded79842a56fcb", size = 81237, upload-time = "2026-10-11T04:52:28.978Z" }, ] +[[package]] +name = "pyjwt" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, +] + +[package.optional-dependencies] +crypto = [ + { name = "cryptography" }, +] + [[package]] name = "pymdown-extensions" version = "13.1" @@ -3903,6 +4236,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl", hash = "sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4", size = 126420, upload-time = "2026-09-20T20:59:04.025Z" }, ] +[[package]] +name = "pyperclip" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/52/d87eba7cb129b81563019d1679026e7a112ef76855d6159d24754dbd2a51/pyperclip-1.11.0.tar.gz", hash = "sha256:244035963e4428530d9e3a6101a1ef97209c6825edab1567beac148ccc1db1b6", size = 12185, upload-time = "2025-09-26T14:40:37.245Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/df/80/fc9d01d5ed37ba4c42ca2b55b4339ae6e200b456be3a1aaddf4a9fa99b8c/pyperclip-1.11.0-py3-none-any.whl", hash = "sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273", size = 11063, upload-time = "2025-09-26T14:40:36.069Z" }, +] + [[package]] name = "pyroaring" version = "1.2.0" @@ -4223,6 +4565,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -4577,6 +4928,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/35/1cceccc5fcb50fa2ed53e2aa278cd032f3902682a73e763fb1ac3be8e6fa/rich_argparse-1.8.0-py3-none-any.whl", hash = "sha256:d2a3ce7854654e2253c578763ab0a32f05016f23a55fadba7b9a91b6c0e92142", size = 25616, upload-time = "2026-05-01T15:18:42.395Z" }, ] +[[package]] +name = "rich-rst" +version = "2.2.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pygments" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/cf/0e/faf7c7e36630561e3e9611730c47510cd972dd5fde8f95941ff78f76accd/rich_rst-2.2.0.tar.gz", hash = "sha256:b1e6a67f8f694a6f36035624bf73e2b1a0a4be13edaf3ba5e654d9758b61073a", size = 326270, upload-time = "2026-09-27T19:12:42.163Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/4e/716b42bf066e180c1235783b46f095a5d871d05dca7006d2295aac29c491/rich_rst-2.2.0-py3-none-any.whl", hash = "sha256:1ea1c43813dc4a8d86475fce27ffb74e0ada3d4656c9f8130ba2a59868055fe9", size = 279274, upload-time = "2026-09-27T19:12:40.548Z" }, +] + [[package]] name = "roman-numerals" version = "4.1.0" @@ -4718,6 +5082,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, ] +[[package]] +name = "secretstorage" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "jeepney" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, +] + [[package]] name = "semver" version = "3.1.1" @@ -5157,6 +5534,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/68/e351109f7732f193133c67fbf104178f79b534a925db8f69c074fd190c2c/sqlglot-30.31.0-py3-none-any.whl", hash = "sha256:51a876a43edecf8ee6fd4c872cf6713d899c4234256bba0be96a8c7c8d8781e8", size = 811981, upload-time = "2026-11-10T18:02:49.108Z" }, ] +[[package]] +name = "sse-starlette" +version = "3.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "starlette" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/41/43/a232e3cd804293bd4f6d01e8895bc0f6f83ea148d712baf9d783c7d93d23/sse_starlette-3.7.0.tar.gz", hash = "sha256:326416da2ab52e9a41b3f409d3ff897eda45aa68c9f3b7b17d0913aab8d219c3", size = 43869, upload-time = "2026-11-07T11:15:49.389Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6a/11/25cabac0a24fa76f58637d8671b549c90d818d69d26beecf0c59a3b186e0/sse_starlette-3.7.0-py3-none-any.whl", hash = "sha256:f1d3136caa12d129284fca3d2ded79bc9af943320b6472712955fea52a6d3eea", size = 18432, upload-time = "2026-11-07T11:15:47.86Z" }, +] + [[package]] name = "sssom-pydantic" version = "0.6.10" @@ -5187,6 +5577,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" }, ] +[[package]] +name = "starlette" +version = "1.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/bb/3d8757f21e9a53d9035c9eb039a777047139a16bfdbedfc7a58241c3d914/starlette-1.10.0.tar.gz", hash = "sha256:b8399098d4c1e3522734f7235dc79b1fc967f96cf139c81e0fa30479e0685f9b", size = 2734356, upload-time = "2026-10-24T15:04:12.342Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/88/2e/947cfe02def9371c07cd5bf5b6b61c829e9aecb9904db02a66cfafbc1eec/starlette-1.10.0-py3-none-any.whl", hash = "sha256:478c0ab3b4ced6e82719849c118fa406f0278a3499c1a898cd15cd31bdc38ce4", size = 80571, upload-time = "2026-10-24T15:04:10.371Z" }, +] + [[package]] name = "strictyaml" version = "1.7.3" @@ -5483,6 +5885,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0a/66/9a06c2227fedfd4919f0eaaa8b94377d9730882a7e4f95823c9a331ab5e1/tzdata-2026.6-py2.py3-none-any.whl", hash = "sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec", size = 348210, upload-time = "2026-10-28T09:57:07.276Z" }, ] +[[package]] +name = "uncalled-for" +version = "0.6.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a3/60/993a9757dc4a41727351828dc784e66fd467ed81d46cbec25d44cae69657/uncalled_for-0.6.2.tar.gz", hash = "sha256:58835db667a85408c7b6d6ad8dfdd74d7713d3be669b46396ba3d2c6a889242d", size = 73820, upload-time = "2026-11-19T01:15:30.596Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/fb/81dabcc9c8f4200eb5eaa600458978303f1a84ffa5ed356f70bf6c5d1f04/uncalled_for-0.6.2-py3-none-any.whl", hash = "sha256:bcac08897ceb7113b52e6c8070eead4da1610ae23e6e0d0e9f2a6c4bba7905cb", size = 20127, upload-time = "2026-11-19T01:15:29.148Z" }, +] + [[package]] name = "unidecode" version = "1.4.0" @@ -5510,6 +5921,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, ] +[[package]] +name = "uvicorn" +version = "0.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283, upload-time = "2026-09-25T06:52:37.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427, upload-time = "2026-09-25T06:52:35.829Z" }, +] + [[package]] name = "validators" version = "0.36.0" @@ -5573,6 +5997,45 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c0/aa/7a7210433e000f2491ddec7756d312009b0a47847e5c3457ccb03852cfdf/watchdog-7.1.0-py3-none-win_ia64.whl", hash = "sha256:0cfc5a6b3d4e76fc3e18fd371ee79d57945004da4105b589f71bb16966fd3e67", size = 103256, upload-time = "2026-11-05T09:33:20.159Z" }, ] +[[package]] +name = "watchfiles" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/68/e6aa0b77d217b31f8f486ec0cdfe5e00e6e38dc0be657e7d85819b9faf0a/watchfiles-1.3.0.tar.gz", hash = "sha256:99aee4a07847c06820765fd7b1b49ceac4f3f711ccb7d104655a33231de1c207", size = 108119, upload-time = "2026-09-21T09:08:55.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/fa/c0b840d5d8bafe640925408ce11095948c8945bb61be1e67d0ab629b872a/watchfiles-1.3.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:000b9688fc8133037a8b075c8ebf98f32844ff8964dda61e85c1db547dafc441", size = 399703, upload-time = "2026-09-21T09:08:13.518Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/894799b485fe9473ad10422a0d9668e53fb78e3a2b5cc6061159572844ad/watchfiles-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:bbc1198edfdc90fda0600f825aa94150f428dfcbf8138746f55998e0e660d64c", size = 397647, upload-time = "2026-09-21T09:08:15.16Z" }, + { url = "https://files.pythonhosted.org/packages/5a/61/f277cd5ead05f5d37b98b41fbdd0c943ee0c04d49738f62e902cb291cf43/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4af9c464d410e8c44b58ddf1cbec7c3a02c1cb2c6c80df9c74659ba2fbda0ab5", size = 454994, upload-time = "2026-09-21T09:08:16.587Z" }, + { url = "https://files.pythonhosted.org/packages/46/66/ea4c01382975e53b1ac1b0a0ea1b353416b8127fb6e04f5c1c313119aab6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb94e40b9a0db19636b5e3fed0a9803452ecc7381df13b385272abb779ee83ca", size = 460105, upload-time = "2026-09-21T09:08:17.961Z" }, + { url = "https://files.pythonhosted.org/packages/31/c3/1a501b597817096c9831c255f6270c124ce0e2faeb74124be24dcc93b86d/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7398930a2b76b9dc67a4e6b8bfc7baf30946892a86f417a3920c07bdd6febc30", size = 493761, upload-time = "2026-09-21T09:08:19.327Z" }, + { url = "https://files.pythonhosted.org/packages/5a/16/77352d05e152c7786a016e9f1decdb4507e8aa5bcf3343c72142a0413d14/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:025b108f2d5cc2799cb941f79380df3e8b22bfb589c218e801767fe78d598c95", size = 577500, upload-time = "2026-09-21T09:08:20.723Z" }, + { url = "https://files.pythonhosted.org/packages/95/fe/81001c55b24466fe9911135e50269fd3a29f43e446627a92adfab54553b6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9ef3cd403d756a304a0a08a112163c63da7ec72c8c9d083c98818291ff8f29", size = 469209, upload-time = "2026-09-21T09:08:22.346Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cf/e3ad894ed5909a7c44b0707e89bc25d19a3d78b3a7aec846119443b6ae56/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5768b49e426fd5b550b012c866db347cdf15c398ef98dc557b6e6b72fa74cd1", size = 458353, upload-time = "2026-09-21T09:08:23.7Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f5/2f106829e6e2bc91ad74614730f6df484202b9a4d20d927dbf2f08d37d44/watchfiles-1.3.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:d36c72fcdc08f143d87316a4c4c3b064e340da2c7383d39803950740ed418935", size = 468707, upload-time = "2026-09-21T09:08:25.167Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/2741ab776b978669c1f3bba1e573f1a184f727ab8b4d3a6d9c61090800ed/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c9865240e065a2247f4b5248528ae3d01e0e8ae071462c250c884dc16e3ee3", size = 632510, upload-time = "2026-09-21T09:08:26.532Z" }, + { url = "https://files.pythonhosted.org/packages/48/64/db4a4684275fe9b696ee5118c194705c1423b3d42181c3f8ee70194d5c73/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:380f513d26cc2e598266b88d66456e07d67ef4be8f0f435a1154d9c44b43d509", size = 661723, upload-time = "2026-09-21T09:08:28.004Z" }, + { url = "https://files.pythonhosted.org/packages/52/80/95ddb24f6a2f595c031c24432ab7b8aa7d6d1ada0c6d63a552166e06e8d1/watchfiles-1.3.0-cp310-abi3-win32.whl", hash = "sha256:509d9f74d2bec5c1f4cd868ddcfe0c9601f7ea53066dfcb19d8ede742c4de661", size = 277761, upload-time = "2026-09-21T09:08:29.335Z" }, + { url = "https://files.pythonhosted.org/packages/48/a1/1c4a1b3c8030cd03206232612be8a6e8799093490c220b0fcb935d02b75a/watchfiles-1.3.0-cp310-abi3-win_amd64.whl", hash = "sha256:1acabde19b67e673274e89a04d613b7fb1f122e2c4b8ec9a581e59c149264b61", size = 290643, upload-time = "2026-09-21T09:08:30.45Z" }, + { url = "https://files.pythonhosted.org/packages/82/e0/d2db577c632d26d08078c4390dd870b8555c4e558df25d22270f0166c71f/watchfiles-1.3.0-cp310-abi3-win_arm64.whl", hash = "sha256:d978cc1dd7ba44f5590d7de74f7a9c8be7262d9e469c2c0efb8bbfc2574321dc", size = 285510, upload-time = "2026-09-21T09:08:31.561Z" }, + { url = "https://files.pythonhosted.org/packages/4f/55/a122743607939429ab6a7272672f49303aa02bbd34c3f06676765a3861de/watchfiles-1.3.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e0204f90677b7fb8d3a824279a71a01166bea9f6e53923b8056592d87b33b0b6", size = 395978, upload-time = "2026-09-21T09:08:32.747Z" }, + { url = "https://files.pythonhosted.org/packages/3e/9d/0ee1b0ea42acfe72afa2484f0222e2386dd09992e9034a81e63cc3cdab0a/watchfiles-1.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7c1125e99f84934a92996ae343501285617373acd66f16b59fd5654b27ad8d80", size = 387997, upload-time = "2026-09-21T09:08:33.953Z" }, + { url = "https://files.pythonhosted.org/packages/b7/f2/4319c6b683e8bd28465dfc53c750617a2a07f072eb2e3ab0182ccf812229/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a85e243c87109d9f48a2e0d10d6abdee8768cc114848b32281ec83d7f81d86ef", size = 448067, upload-time = "2026-09-21T09:08:35.104Z" }, + { url = "https://files.pythonhosted.org/packages/b7/71/9f37ac192a51a24de3a2d7fe9aac1e99b2b2668300674ff7a9f85d95beb1/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd538c71766c59c732e2c5cedc39323c99c11897b99f93eac773438e54af206a", size = 452881, upload-time = "2026-09-21T09:08:36.238Z" }, + { url = "https://files.pythonhosted.org/packages/34/07/1224667c32a4a96876ddd642da69733cbfe6a15897db0865aec5a882cebc/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da213822ec9082b62cadf3008e07a8c690ec7a3c89f6dc5e55a8cf07b56a1b26", size = 484802, upload-time = "2026-09-21T09:08:37.717Z" }, + { url = "https://files.pythonhosted.org/packages/bd/4d/7f29cd87e1f826df86fc0418d0b8f370ca0311c29c618406dc492aad9951/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4593392e87669836670f87d62fc754059eaeb0e6157af44060a46558cebe5a3", size = 567522, upload-time = "2026-09-21T09:08:38.94Z" }, + { url = "https://files.pythonhosted.org/packages/e3/ac/3aa1f84abd4dbbfbae99a817dd931f77b439261c911189cad4c30fdfd470/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26b81bc515a0f03f66a69a6fea86eea54c8eca4173c46dd7222569f79dd2f977", size = 461997, upload-time = "2026-09-21T09:08:40.142Z" }, + { url = "https://files.pythonhosted.org/packages/82/1c/27c66764d5b0dbb70a36770a32eca217eba739c28b24e8583a035087a3d7/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca39661934749df580d89f3dde266f2af7c6968f4c21c72ce2dc387d299e6aa7", size = 451856, upload-time = "2026-09-21T09:08:41.305Z" }, + { url = "https://files.pythonhosted.org/packages/1e/b3/a45827084d953c1d0dd3d75abf85dcac26a0f5d115ad2fe0b7e9e931573b/watchfiles-1.3.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:04c5500faa725d69a99b0f63750e80795e455bb951849e9c98a811b4324997c4", size = 459889, upload-time = "2026-09-21T09:08:42.58Z" }, + { url = "https://files.pythonhosted.org/packages/7a/3d/07740d23ce2bb23419bb24055c435a9abe5fe03efdaea1ba0720b831aab6/watchfiles-1.3.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:ec3cd4bb181a7b6329134204c2277c7144ab5f62cce9c5335d6f901ee7ce8d6a", size = 624482, upload-time = "2026-09-21T09:08:44.064Z" }, + { url = "https://files.pythonhosted.org/packages/91/46/9540f8123eaec86df989f26d7704cad0448d9fb7271763a4f8f9fe23617d/watchfiles-1.3.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:e5871d4a7f788a9f64fb06f5793389d29d171e5c32d37c3ebfd520b017cc7694", size = 653724, upload-time = "2026-09-21T09:08:45.257Z" }, + { url = "https://files.pythonhosted.org/packages/25/9d/b9deb52f1e4528ffe44338c5ebab95c32ed79203082da661de50288e130a/watchfiles-1.3.0-cp314-cp314t-win32.whl", hash = "sha256:81c989d7267bfb7b676cd32c83bec97914e396870983cbf493e29415037bff5e", size = 270863, upload-time = "2026-09-21T09:08:46.518Z" }, + { url = "https://files.pythonhosted.org/packages/59/59/8b1881afccbafda06248be4185e28034dd73c80c85e4da24f2cb0c1321eb/watchfiles-1.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d754e049006e81a9be49dc78ba7802081cee426e968cbc63ee19e164fa2d0e39", size = 286879, upload-time = "2026-09-21T09:08:47.874Z" }, + { url = "https://files.pythonhosted.org/packages/46/dd/41c12ae9745d5d72f709333287679d71cbba68ad7678fed872207bba94f4/watchfiles-1.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:a5b631db08fd3032db57ec6a92d1777cd60f88803287e4e941a4858c53aab1e0", size = 276924, upload-time = "2026-09-21T09:08:48.976Z" }, +] + [[package]] name = "wcwidth" version = "0.9.4" @@ -5618,6 +6081,115 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl", hash = "sha256:032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d", size = 14905, upload-time = "2025-10-31T07:51:01.778Z" }, ] +[[package]] +name = "websockets" +version = "17.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/79/590c5f0cc7d1661195aeefccb0e342b79f4c77d368d526d028821a390332/websockets-17.4.tar.gz", hash = "sha256:09d3939b70df0c8ad2da145ebd50978bccaee1d565e838b0e7921a34d38571eb", size = 191391, upload-time = "2026-10-30T15:10:35.873Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cd/58/eaf27fce59e3f5b9997cd7c77749ef214e077daeffac95479be6ff9f5cfa/websockets-17.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f24bed98595b9d854da8d6debaf82754abf3e8fe63ec90505ef560566814df7e", size = 218769, upload-time = "2026-10-30T15:07:42.558Z" }, + { url = "https://files.pythonhosted.org/packages/13/8e/287eca6d399a953253492189153976676994dfba1b7e82f0f992d14c1404/websockets-17.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:156221ad7638b5abccefd6f798916bf3f8ffd6ac51480eb4ba5b05df914ccbf3", size = 216454, upload-time = "2026-10-30T15:07:43.874Z" }, + { url = "https://files.pythonhosted.org/packages/b3/65/098fd4d221d691395492fcba88c895dde19a51d371b06b8d0be71d3f781b/websockets-17.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:74e46409666c358b93fce0d8c6f4cb107d8dd7cba1302a38d49adf6fa8193c7e", size = 216715, upload-time = "2026-10-30T15:07:45.409Z" }, + { url = "https://files.pythonhosted.org/packages/e9/29/9f9833bc83c1bf2e5597d1ef374d3ccd6467b9b45400cff399c85331ef6c/websockets-17.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e3e80cad85ba68c07c1e8d4ccc9cf51e63bf6a73d6bf631640be947bd72b6b63", size = 226096, upload-time = "2026-10-30T15:07:46.929Z" }, + { url = "https://files.pythonhosted.org/packages/ef/75/6a95f93661bba2f1a4ec82eb26c6d30e6ba253569ecabb7a675139db6051/websockets-17.4-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:25eb661d8ba7e2b85ebb4e7a1bb9f6a3280910ed10b84330a7abfb5ed32d2764", size = 226382, upload-time = "2026-10-30T15:07:48.332Z" }, + { url = "https://files.pythonhosted.org/packages/80/b0/b0d9427dde0609e2727a9a966fb5fc5c6ef6aa7eba4cfc98791f8b7f1b75/websockets-17.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c74954409ea5789f7a60daa5f94580c00ad0b60b20751c47b35c4784877b5057", size = 227617, upload-time = "2026-10-30T15:07:49.799Z" }, + { url = "https://files.pythonhosted.org/packages/3b/00/fb1016d850d61676d1732434e2361d116c1de369078e59d21f4713115a19/websockets-17.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:47beedf07a706140b50786457e082497ee6b5970c41e49bcd1572ee3f4c73f80", size = 229054, upload-time = "2026-10-30T15:07:51.258Z" }, + { url = "https://files.pythonhosted.org/packages/77/c4/bb666e196541e42237fdb625303ad099005cc835c4e2e0a5d97bb9c4185c/websockets-17.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e282f35ece8895b9fa5af406fc6a4cb42f01e1009525cf46641de584781c83b", size = 228243, upload-time = "2026-10-30T15:07:52.975Z" }, + { url = "https://files.pythonhosted.org/packages/6d/f1/32ae0f45d8464e1606ef3f6a54318d8754b27889de18bb1b2176a19bea39/websockets-17.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:361f9a21a33191d5287e7a75f508fc3b9bf47aaffb0241198d6b7be3384f294e", size = 226995, upload-time = "2026-10-30T15:07:54.574Z" }, + { url = "https://files.pythonhosted.org/packages/f8/9f/5680151ef10e0bfb6f3bf09a992ce6c3e93bbb84fc8ec5d03ad570ac6532/websockets-17.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4073ab7cefe5a2e7dc368c1f01eb27716dc8ea8f1c69e9b7725ebce05aa53cf7", size = 224244, upload-time = "2026-10-30T15:07:56.158Z" }, + { url = "https://files.pythonhosted.org/packages/e2/66/409117b6882b91960598d62b95fac33d70a7aa2e1e1e854c6b64e76712f6/websockets-17.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a50bd9b44dd9b7b845802030b4d88610c01c7f95571b4cbd3a1aeec17a34c9fd", size = 227058, upload-time = "2026-10-30T15:07:57.539Z" }, + { url = "https://files.pythonhosted.org/packages/35/e6/34cab5913bea9d011ae2592a7672340179ff14fb98e889e5e41b071d8134/websockets-17.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:ad2e9e152bec6c4f7e4cdb4adcea6b708400090eec04bfe5ee5b30d368ffd721", size = 225654, upload-time = "2026-10-30T15:07:59.044Z" }, + { url = "https://files.pythonhosted.org/packages/a6/b0/6cfe18e12f23618a3d69723c828037c1282fee7ffce006ae7b5cc0e58ca4/websockets-17.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:5830f94a7190a33e55d06d89242ea951ee2009c3a27fc31538f3ac9386b27edd", size = 226425, upload-time = "2026-10-30T15:08:00.492Z" }, + { url = "https://files.pythonhosted.org/packages/c9/71/14d31ea34bafc07c6d96b362a44b301b9a0f2fb260490a8a6cb7db682bc4/websockets-17.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c0cc2cb47face4dae64bd70a9268dbebddb3e48c1b48cf6d7b6ac1471609fd88", size = 227532, upload-time = "2026-10-30T15:08:01.932Z" }, + { url = "https://files.pythonhosted.org/packages/28/69/df3398ca1320a8b76e57ee41053d5c09955383cf60888ffa5771f51a4cc8/websockets-17.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d0b6a9bd86f865ebc9d847a1eb0a655a0e2b447cbcde743dda97c4298415cc0b", size = 225090, upload-time = "2026-10-30T15:08:03.328Z" }, + { url = "https://files.pythonhosted.org/packages/d6/94/06afe778cf9893e9ac10209c0d4c70aaf0b72289245a8b7efb92cc80927c/websockets-17.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:7a7e9321822ca59ba56fa8744c3e3f01ef74d547aafe0d383d000d7a4d11eb79", size = 226041, upload-time = "2026-10-30T15:08:04.714Z" }, + { url = "https://files.pythonhosted.org/packages/c6/7a/28e9822dbf0bc63341dc45d8d5f1f1f7f375991fc8895bc731105fbde371/websockets-17.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:de29795f23833cdfb68ba45c8a5cc0ec49778ea5deb670188f4465bba5930d25", size = 226323, upload-time = "2026-10-30T15:08:06.103Z" }, + { url = "https://files.pythonhosted.org/packages/5a/d8/606c9c404e82731e4520f6397c8c16549e9cbd39ac583e7ea7d8972a6ae7/websockets-17.4-cp313-cp313-win32.whl", hash = "sha256:37ed7782972545e67d97ef9a172e9499d25459b1e32cb99d1c2a30c572f2441b", size = 219248, upload-time = "2026-10-30T15:08:07.617Z" }, + { url = "https://files.pythonhosted.org/packages/60/a1/17e507dadbfccb755758ba977b165fec92924dfc4a8d2573b4fc1561b0cd/websockets-17.4-cp313-cp313-win_amd64.whl", hash = "sha256:203e42eb77e06f85778a908b34c5ad032f34e98a68ef33920d6685c1e65a9420", size = 219560, upload-time = "2026-10-30T15:08:09.108Z" }, + { url = "https://files.pythonhosted.org/packages/01/ba/4f51fe8db73889ea3ecfdeee283d4ac3b7ad7a609c0e4c1fc6de04078021/websockets-17.4-cp313-cp313-win_arm64.whl", hash = "sha256:d8255971f6789cf0cb365e3e22b9b166eba978548a26241eaa5b7af0440d885a", size = 219496, upload-time = "2026-10-30T15:08:10.574Z" }, + { url = "https://files.pythonhosted.org/packages/1f/5b/085530961668661300de7248f9bf3ba84e98f47875f0fcc87637d024f82b/websockets-17.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:d0f943b53bb1d7389d72a5380078670e5fcc34a1328142148b72d20fd82031c7", size = 218810, upload-time = "2026-10-30T15:08:12.14Z" }, + { url = "https://files.pythonhosted.org/packages/f2/fb/5a29e9569f74988822b7bc076287dd7f9f5b68a2070cf5eb4cfada5863d8/websockets-17.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2fd562372d4afaf18d1c7f1c68e39bd32818bb16d5f57f778a4a4ed88eedcaae", size = 216506, upload-time = "2026-10-30T15:08:13.561Z" }, + { url = "https://files.pythonhosted.org/packages/df/73/eb8544367c13cd87df39d41b74853741fb7182e2a4bb659ad7d36f9066c8/websockets-17.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a0457561e8a87a6a3b509e21c35084d7d207ced262139de3f738e9676a476bbd", size = 216719, upload-time = "2026-10-30T15:08:15.08Z" }, + { url = "https://files.pythonhosted.org/packages/24/fe/c3707e426d9bd779dfad9f319b2b431e5e60d563d7547d3780cbd1646028/websockets-17.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7b4ee34acefa9ba1333f02b817982f1e4127f6f9b975017a707c4e738d7f9fb9", size = 226098, upload-time = "2026-10-30T15:08:16.581Z" }, + { url = "https://files.pythonhosted.org/packages/c1/24/d203439f909ffbf37260ca28a3432381412d2e79912ebf98be572e63055e/websockets-17.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ef03fb4b3685df0ed9dc6a074a090f030345cad2c23c61818da3e08e693a9caa", size = 226449, upload-time = "2026-10-30T15:08:18.29Z" }, + { url = "https://files.pythonhosted.org/packages/4b/17/3ae5d9486a170a222dd0437b4f2bc02ecf324177498a942dcac200e65d05/websockets-17.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5e970d3996c597186c9dbb6e77a8048737a834ac03cb13186a54999e0e1c31e", size = 227695, upload-time = "2026-10-30T15:08:19.692Z" }, + { url = "https://files.pythonhosted.org/packages/34/37/1307a8e30a961726f2350982fc01b0b3498db29b030c78c566e65ba64487/websockets-17.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6afc32a7a1a40b071070419537df3559d26907086556769683fa97b7b6e90fd5", size = 229069, upload-time = "2026-10-30T15:08:21.682Z" }, + { url = "https://files.pythonhosted.org/packages/9c/bd/4c6167744e202d20a3c770bcde8fae5d427317b1885712c2e02718cd8133/websockets-17.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5da7f7f1ecf866186f195f7ecc374c915e1838496b86f2f3b41da27c798a25c3", size = 228296, upload-time = "2026-10-30T15:08:23.371Z" }, + { url = "https://files.pythonhosted.org/packages/14/b3/3438eee41b39afb44c3c6e45dd686b88a7f8d742b96877b77de4fe0e0255/websockets-17.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:043945a43873c7c11a3f75238d475c95f902e967de329eb8763704546cfaa6b6", size = 227045, upload-time = "2026-10-30T15:08:24.749Z" }, + { url = "https://files.pythonhosted.org/packages/46/39/0bac9052d402efaad1b202ff2f1a57b6334aa624d20f509312765601b507/websockets-17.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:927d602fb8fb595a69e6b1ae998a6ec19851d0fb9391759456e6425090996647", size = 224297, upload-time = "2026-10-30T15:08:26.275Z" }, + { url = "https://files.pythonhosted.org/packages/a1/7f/5809f6ba5a7d1065301800daf605bdb661a9db5ef868ed189dbf691122ac/websockets-17.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:769e4f7b090e10cae4c082e3e39b9458ec2444505df84953c16c0468b16539c4", size = 227169, upload-time = "2026-10-30T15:08:27.703Z" }, + { url = "https://files.pythonhosted.org/packages/fc/f7/c8f6771e7cc2a038cf78bb12d04d29dbbbabf2a84d76999bcc94532f3020/websockets-17.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:03ddf4eb76ced8725dcbb000ff60b0fb8d7330ed011f6dc725a1925c5fe02051", size = 225632, upload-time = "2026-10-30T15:08:29.441Z" }, + { url = "https://files.pythonhosted.org/packages/2c/2f/9104786d952978ca2d011321109e35c05be24c3937ef2a62b4d6e25f7515/websockets-17.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:94622cf24e80d03054ceb69a06a86c950e12fe993ed06b933f7895232cacbc3e", size = 226415, upload-time = "2026-10-30T15:08:30.962Z" }, + { url = "https://files.pythonhosted.org/packages/ae/a7/2e1606b3d616b86bb6c70a62d5f87fc9a3dbd81f0bbe24c4706ace76751d/websockets-17.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bc581579885728bb54ef15000295c20e6a64a200b1b73d5cfca71879a3eb3e1f", size = 227594, upload-time = "2026-10-30T15:08:32.443Z" }, + { url = "https://files.pythonhosted.org/packages/67/2d/78ca44e2be6d397aee63a4e68d3a99a33bb14bb3405e94688bc5a2160db5/websockets-17.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f42671900e0b602d05a074c52fd3d757c2f8e5c4f73f771794dae1aa4aea5f0e", size = 225146, upload-time = "2026-10-30T15:08:33.888Z" }, + { url = "https://files.pythonhosted.org/packages/d9/b6/6dad684e54fc8e09d70c9a095506db5c836207d8e846e3cb1e79424e3492/websockets-17.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:2b23dacf7a8a88afe92f77bcdbf4cf043a84692bac2f2da02f4011c687e19b48", size = 226095, upload-time = "2026-10-30T15:08:35.285Z" }, + { url = "https://files.pythonhosted.org/packages/d9/e5/7b04bad7c11c1f92238429352defbac1406743338049afe1328c7a7f95c0/websockets-17.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d48795c7d64ca1c42e3d61fcc6478cb15a19b6ad3f9c882bc8d8dee49d4a420f", size = 226395, upload-time = "2026-10-30T15:08:36.8Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/4eb29c4b12f77b3139470063308ae1e6a0860b9073b1d27d56e8ce19c512/websockets-17.4-cp314-cp314-win32.whl", hash = "sha256:8588edf317fbe4f16a949bc5a6ccdd22b6e6fd5b4745d128bc40bccf31024ce5", size = 218784, upload-time = "2026-10-30T15:08:38.263Z" }, + { url = "https://files.pythonhosted.org/packages/6d/85/2496a24de80c215b55a15f037bc368a2ed0ce7ecb1e909039ff4e76395de/websockets-17.4-cp314-cp314-win_amd64.whl", hash = "sha256:70293c25d79d1f084fdd665894481547575e429d2a75e8d72caec2f6b6b07af8", size = 219137, upload-time = "2026-10-30T15:08:40.112Z" }, + { url = "https://files.pythonhosted.org/packages/9a/a3/cfe95d4eb569035e5a1d2e5cbeb4ccf0eb15a19e1ef2ad819a85960bf375/websockets-17.4-cp314-cp314-win_arm64.whl", hash = "sha256:adc30e5d064c9e33049e825408fa4e952f4e8a0cf4c329dc2ded72e1bb5bae08", size = 219017, upload-time = "2026-10-30T15:08:41.778Z" }, + { url = "https://files.pythonhosted.org/packages/2b/dd/4143048a2b27303e7de8ef53d7d0ee3987734fbf93e09491d5a1b0a55187/websockets-17.4-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:bbb6bed49f083e2ec8ae0bb21c00c7c5171ab8e08a8b008c61a43fc6c944fdac", size = 219141, upload-time = "2026-10-30T15:08:43.184Z" }, + { url = "https://files.pythonhosted.org/packages/a8/3e/a63b5f3a5c24be2898eb22a3093a56d4ac4cd24ae79e7ab7a1b971ed4611/websockets-17.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c7a56140fcbf4662cb263754a754c651a16fe2196cae1fd232d18f89acd30c83", size = 216675, upload-time = "2026-10-30T15:08:44.682Z" }, + { url = "https://files.pythonhosted.org/packages/d6/80/f238a10f0b23853c665c602a42784023995ac26674e480252c21af39f4f8/websockets-17.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:18d3710becc612ec6f0229425bd6495133dfcbf30f7b6569585acc8ed2f1b452", size = 216880, upload-time = "2026-10-30T15:08:46.304Z" }, + { url = "https://files.pythonhosted.org/packages/d1/b1/899b69aba1e8d65ac5bfe90b97ada22a815f1dcd3840bf188104738aac33/websockets-17.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:00ad98e4bb0df7a6711c645eec16d5a9fdf99f0692bd4f5b6cb85a68677ba8df", size = 226462, upload-time = "2026-10-30T15:08:47.723Z" }, + { url = "https://files.pythonhosted.org/packages/28/a8/598c2b2a22dfe14d352400df804db8b78a68c1498b3f775a658f07f21618/websockets-17.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7849a096d6fdba3f2bdf849eb89f2d233a24400c92faa3efb25fa8dc615b706c", size = 226741, upload-time = "2026-10-30T15:08:49.314Z" }, + { url = "https://files.pythonhosted.org/packages/6e/ad/84a6de3cc3dd4ee336428c31f89a82e002299f83e10d967f0e9a5cbaf591/websockets-17.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e30550965c78223439815dbdf40c93eb8931dfbf12427110dcfcaeebef76feb6", size = 227914, upload-time = "2026-10-30T15:08:50.746Z" }, + { url = "https://files.pythonhosted.org/packages/69/ab/1e191b97d60a934c071b7b73c39c5db39721a925cc53acd929c615f85550/websockets-17.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:725be1aff55f1c873a79cea0f612df9dff21a9d2c2e05783665b824b8aa7c24e", size = 230059, upload-time = "2026-10-30T15:08:52.596Z" }, + { url = "https://files.pythonhosted.org/packages/f2/57/ef8a64732ef4367091ab05f9b22c1ee4c28657d4a0d088a644198eb5ded6/websockets-17.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b6c7f91520c29420639925fd1cecceac9ca07366b50c09ea24eac1cd116b4866", size = 228486, upload-time = "2026-10-30T15:08:53.967Z" }, + { url = "https://files.pythonhosted.org/packages/48/1f/dbad3afcacf2c52d1fdf83b611d849598f0a047d7333ff476266532877b8/websockets-17.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6531540aa63e6864cb0615a3a5da70ae03191334507852a5385320bb7916df3f", size = 227216, upload-time = "2026-10-30T15:08:55.687Z" }, + { url = "https://files.pythonhosted.org/packages/5b/47/59e325d6ca3e04ba5cc7a139d891ef83cad3fb460a183c8f359f378b69a9/websockets-17.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a57b99f2e28922b5a2434317580fd23fdf118caff4c5b752f5452836426205a", size = 224755, upload-time = "2026-10-30T15:08:57.203Z" }, + { url = "https://files.pythonhosted.org/packages/b8/b6/28a471318dfb0060b58c47d0a252cb5078528a1aa1fdf5e6e88e2a61cf18/websockets-17.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8fd30d217a4275107667fe1d4f601e2d50c797d0727148cfbb384536d65f04d3", size = 227508, upload-time = "2026-10-30T15:08:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/ee/29/7485ac6fc4b342b43964811381f56276fec967cae6a851ae4b815613b10c/websockets-17.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cc0cd4065a65ac98f3e2742ee9f314da9d224db49b07dc17d3a20cef0d79658d", size = 225961, upload-time = "2026-10-30T15:09:00.414Z" }, + { url = "https://files.pythonhosted.org/packages/95/a6/520e1f7aa9644ddf58f5a51fc37f46e15293a68cd35b70d5b81b7331ca1e/websockets-17.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:71f76bb8e59be847330b8a8d3753f85397677e128f9f9692c215f8cf721cefc7", size = 226851, upload-time = "2026-10-30T15:09:01.979Z" }, + { url = "https://files.pythonhosted.org/packages/e5/0d/444799abe7418f361da8c201c6858b64b4e8d47265f90851e5cb68b4d90f/websockets-17.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c2db7b9e2d726f7e051bedb67abcf460b84b11437b2f9fdacda7e6931dace85b", size = 227869, upload-time = "2026-10-30T15:09:03.405Z" }, + { url = "https://files.pythonhosted.org/packages/cb/86/6da20322a07364a0ab0787a2b0d4367770f59404fdab6e35122413a7f63e/websockets-17.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:332b88022d0acd47c44158bcd43f47f14eaddcd979247f242b1a33291c45cdd1", size = 225562, upload-time = "2026-10-30T15:09:04.891Z" }, + { url = "https://files.pythonhosted.org/packages/2a/da/53e44d96238dc207011ad5d93b0540c58e1c680a86f48dc3fa8e5138b343/websockets-17.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:5cec3e7fbc911ff4fcf43b24aaf1b96906381ad36030f42897def3ff4b78729e", size = 226334, upload-time = "2026-10-30T15:09:06.42Z" }, + { url = "https://files.pythonhosted.org/packages/18/08/82cea972a87f229ee1903e9adf74c17d1e6c4371d581ebb11ccbed4e3a7b/websockets-17.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cbdf4b42f4514422051889424fd03bc269be8c5d751bf39ec29ad761590bd10d", size = 226767, upload-time = "2026-10-30T15:09:07.837Z" }, + { url = "https://files.pythonhosted.org/packages/96/f3/ee6afff802e39e8d6ef29ee4bdc30f8272aaa38295c19a911fad91fe95fc/websockets-17.4-cp314-cp314t-win32.whl", hash = "sha256:16550e24ac22c27e0a728100d8de196b5f054bab6748ef9f0b168fb8b3c32e35", size = 218918, upload-time = "2026-10-30T15:09:09.31Z" }, + { url = "https://files.pythonhosted.org/packages/11/56/b0b07917f9c85a4d24a37300f4ae888bfa7c20f08902dd6a5c61a0d1b26f/websockets-17.4-cp314-cp314t-win_amd64.whl", hash = "sha256:5ec906bb21b35e66cfdcc2ea0133f249f52b7d89e3e45ec8d2bd2d84a0255d7d", size = 219260, upload-time = "2026-10-30T15:09:10.832Z" }, + { url = "https://files.pythonhosted.org/packages/19/a6/a63113909ea602d3d22b0f32a7a9950d8d18b725cfdce735eab9b5f5d198/websockets-17.4-cp314-cp314t-win_arm64.whl", hash = "sha256:c00f0c45851a2c61efffd03dde8fb4d594026b5cc18196c94e15460f1225f2ed", size = 219176, upload-time = "2026-10-30T15:09:12.444Z" }, + { url = "https://files.pythonhosted.org/packages/ee/be/4032fde05e62d1c52edd250fda475e910cac0cd180e377690bff24147b5e/websockets-17.4-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:8c51ac7b31578a7275650f580aac985940985b6d2e57247ac6d5e07f2c936cd6", size = 218740, upload-time = "2026-10-30T15:09:13.833Z" }, + { url = "https://files.pythonhosted.org/packages/74/1c/b47f5786eb45599072411571651606fb22f6e825d522d627daad974ea636/websockets-17.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:023dc13bdc42a55e152acc81c39c90ec6cc4159ed17d23c6d73460fa2a73ab16", size = 216470, upload-time = "2026-10-30T15:09:15.394Z" }, + { url = "https://files.pythonhosted.org/packages/70/98/0021f55de633617b4a8390cca17e700c9ba5d99323103128412dc2c15aca/websockets-17.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6fd8cac4907fec8302726d49e6969acfffaf720bc39b1cee23f473b81f4dce01", size = 216695, upload-time = "2026-10-30T15:09:17.185Z" }, + { url = "https://files.pythonhosted.org/packages/28/56/2c67389b47c4b9e9e9318d9b3e4bb52a2fef3749ff00b0b151a342c4d42d/websockets-17.4-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4441d655a53185ef1b01b1d42ad9b3a96b905110a9219043e3fdaec855ba3b6e", size = 226134, upload-time = "2026-10-30T15:09:19.156Z" }, + { url = "https://files.pythonhosted.org/packages/75/bd/ee3a679a10741a0cdd7aa8bdd6d03999069672826aa7a6495fdf589ed961/websockets-17.4-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3acc45494c08a4d27375a2c28474537f68910219f42624a08011221b04623d04", size = 226569, upload-time = "2026-10-30T15:09:20.587Z" }, + { url = "https://files.pythonhosted.org/packages/c4/42/b293a5d07099549797138d09fda5bf878d302b4980e61678981c8dfdc98f/websockets-17.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b78b39377630a70aa2198e5f03be416e690364a046b1d8af80a9263733083d0", size = 227726, upload-time = "2026-10-30T15:09:22.015Z" }, + { url = "https://files.pythonhosted.org/packages/2d/81/d048ff83c4d0fc386f6008529d12690d0dcb410f73570430f13fe4d543f0/websockets-17.4-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6cf4c002268da5c11cae9083aec71ac943bce9d95e0f83c8bf9a95a3ce421fbb", size = 229426, upload-time = "2026-10-30T15:09:23.444Z" }, + { url = "https://files.pythonhosted.org/packages/e6/21/1df51f94047dfb76c32fd7508036e4b55352d0f04fd0e3bee525a2122225/websockets-17.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ed7fe197b8bb66eef4f62c052e33752d456d2fdda74805469d4bd954950ab339", size = 228211, upload-time = "2026-10-30T15:09:24.902Z" }, + { url = "https://files.pythonhosted.org/packages/cf/13/33d6a28471da4c30a98a2313c76f984be2056a703c5c532225dc28f5dba3/websockets-17.4-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f0494d3d97502e270282cc6a1f8648cd7335b4fe8d5c452006e5b7f9d62b8892", size = 227106, upload-time = "2026-10-30T15:09:26.315Z" }, + { url = "https://files.pythonhosted.org/packages/4a/86/ff8441e56ca677e51f9c826868fccde2a82bafe0ffa337fd38823b64253a/websockets-17.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab9e4bd60796866c578e3afa0a17c98636887827a39c2e238a61989c18fd8e7b", size = 224499, upload-time = "2026-10-30T15:09:28.116Z" }, + { url = "https://files.pythonhosted.org/packages/da/91/470d7a619ce4aa837ff5b6a5836227c43bdfdc42ef837cce48f508992438/websockets-17.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8d148b4b295224d3a19e7bd9f02ebe7344de254b3b2c4d0628ce42ade0bca119", size = 227213, upload-time = "2026-10-30T15:09:29.847Z" }, + { url = "https://files.pythonhosted.org/packages/a4/ba/55ed1422d6c73b5b81996ca195f2a256a339b26c553f5686afc2a926bc2b/websockets-17.4-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:6597a1f7413189b4bc58437babcdf39b79da416e5c477ae98323a88ecc108188", size = 225876, upload-time = "2026-10-30T15:09:31.249Z" }, + { url = "https://files.pythonhosted.org/packages/e9/59/f133d260e7a48d3e231c3a495eeb6114b880c1e96bfc781a949c1f73991c/websockets-17.4-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:a9b0026538719f7a532998d84640ab2e91ca1a051ce4b8d924fdcb607307e921", size = 226403, upload-time = "2026-10-30T15:09:33.199Z" }, + { url = "https://files.pythonhosted.org/packages/45/16/bb3355453ffdc31d3a14ddc0b50e058f13a97d2ed019ba6c380e03456169/websockets-17.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:a703f23ca15a51251fc94f1774683f5aa511588a7ad6bbb83960cdb2a4dcd978", size = 227558, upload-time = "2026-10-30T15:09:34.703Z" }, + { url = "https://files.pythonhosted.org/packages/0c/ab/f3d55a83c2a4daa8bf62594b00a721dc8767518ccb833f932b0178eb6c19/websockets-17.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:6b061ea7ca94966af7cd7a87cb189ff15972c103244611a2b1e8408c04f1dbe6", size = 225369, upload-time = "2026-10-30T15:09:36.204Z" }, + { url = "https://files.pythonhosted.org/packages/88/66/3a320bfb718d9ae1eff15ba7790979c0e6fdec74e388430f426e6647d7f4/websockets-17.4-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:e6d2a185ec4ff0478089305abd833da3a9fda8803eb77325b89ebeae5585a3b7", size = 226268, upload-time = "2026-10-30T15:09:37.565Z" }, + { url = "https://files.pythonhosted.org/packages/da/ee/1f9cdeeaaa2d5e158b7a40f971e74b641b35f977674b46229a5bb14e270a/websockets-17.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9a4e95d56994a45ac76541853796720436ffe462957d0b1b89e87ff3d3b1627e", size = 226530, upload-time = "2026-10-30T15:09:39.171Z" }, + { url = "https://files.pythonhosted.org/packages/da/28/506406ef09e6e168796f0187bdac0767f38a560e6785c4ee26e583aaaf76/websockets-17.4-cp315-cp315-win32.whl", hash = "sha256:b7ea61af293a153f2d67430038cf0efc10140caa623f2bdf5a9f5a457ff20f57", size = 218751, upload-time = "2026-10-30T15:09:40.748Z" }, + { url = "https://files.pythonhosted.org/packages/25/a6/818647a9a9b3ee8bf131e9ab4fa1f2db9c6539f08e7084dd57549e0bd49d/websockets-17.4-cp315-cp315-win_amd64.whl", hash = "sha256:8b45d40f07081b377f0b837ddbfba4a9ab4a89a7b569d3c095436564e767a27b", size = 219045, upload-time = "2026-10-30T15:09:42.251Z" }, + { url = "https://files.pythonhosted.org/packages/3d/b2/0338a9abec3df8556627af987352657ce1c8456f2060b54741570b174122/websockets-17.4-cp315-cp315-win_arm64.whl", hash = "sha256:29b615c2d5dd883f89c107a782681b73a7aa8b2bd894130310e45eb09b143913", size = 218957, upload-time = "2026-10-30T15:09:43.866Z" }, + { url = "https://files.pythonhosted.org/packages/68/92/22e83d003cdbd22be5d35e28c260203e9fbb6c73b17a8e1509f4ad49e74d/websockets-17.4-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:c568e72f5425a74616daeb824afe28da460cbeea036432f41c6af5380b83300d", size = 219056, upload-time = "2026-10-30T15:09:45.528Z" }, + { url = "https://files.pythonhosted.org/packages/2f/ca/071d38e7237f6ada4f577b74f7cbf5bc2ff7e1f9360ebd2837e74a22dfd4/websockets-17.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:bfc86dff091104eec770c29ea1831dbe191968a70aa2d8ae8eda3240d863970a", size = 216630, upload-time = "2026-10-30T15:09:46.951Z" }, + { url = "https://files.pythonhosted.org/packages/fa/76/22d4523f92d88b8f5ebfd3df7ab5e825146af100f7e2caeae7d359ae51ea/websockets-17.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:f5f06f59e19aa0770aaf651ce91c196fa620b35aed49a7e6c36e7335f3f83db7", size = 216841, upload-time = "2026-10-30T15:09:48.285Z" }, + { url = "https://files.pythonhosted.org/packages/87/29/bc94be4a8bc19ef518e9efecd90a992f2991b7cfc6eb8a7430c4e2dc097c/websockets-17.4-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8f32a6cb0b101170abc2d8053af1473a2a29ef2a06d7130994ee8455a50f5dbf", size = 226992, upload-time = "2026-10-30T15:09:49.776Z" }, + { url = "https://files.pythonhosted.org/packages/fa/25/58f69b5fec064764ddf5b88045397433cdb028f5b17c82fbe047fd31fff7/websockets-17.4-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:72d0a731eccf482d5661c6711abfcd5921f3dbfbbb0789061e1918ef4bc89c70", size = 227258, upload-time = "2026-10-30T15:09:51.172Z" }, + { url = "https://files.pythonhosted.org/packages/21/7c/bc932a54c5f1bd78461574aea539fc562fd2de494738a2c0675c4de860be/websockets-17.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48c83103f82fce36191a9e3a8a80731d4340e2da544eae8c5aaaf6a45280e7d0", size = 228589, upload-time = "2026-10-30T15:09:52.628Z" }, + { url = "https://files.pythonhosted.org/packages/c4/92/57ec83066d0bfa8a9b9f594bc61951073c4a86e34c562a5292604118dab6/websockets-17.4-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c8052c3058c402d53602583da5ddcf1a15a63ab1efe0222f47c7be59944e6463", size = 230217, upload-time = "2026-10-30T15:09:54.058Z" }, + { url = "https://files.pythonhosted.org/packages/6e/fc/0f2fe533b5b4dc22ac704b3357f5c99fa60a3837218b3c6c6ad63a60120e/websockets-17.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0c22a9c6b2349204f56123dd0ca7779e989d68e336d0dc02ad823485d44925de", size = 229026, upload-time = "2026-10-30T15:09:55.727Z" }, + { url = "https://files.pythonhosted.org/packages/7c/cc/ed28307835106a9a2dbd1a5efc597e8befa3d0a5f2b46a4c1e466b2f4759/websockets-17.4-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0dcce83837328e6795225ead920d3aa8d867cc9d6922aa558b0e9111599fdbb9", size = 227904, upload-time = "2026-10-30T15:09:57.221Z" }, + { url = "https://files.pythonhosted.org/packages/39/4d/24ad7deddc7b31c3e430acc1c00c7b384ae42f1f973e9cc01aa42d19628f/websockets-17.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4b79ea31cf277c72d2307c6cb8bd32a2c151959e80a2f24fa85df620c5f2860c", size = 225253, upload-time = "2026-10-30T15:09:58.624Z" }, + { url = "https://files.pythonhosted.org/packages/93/f6/9c92230ebfab42fee482b236c45c32ed34b24f7360080b400cce6cfcac11/websockets-17.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:3d27153a0413696767b98b5d96173a2a7e27cbfca02000a6069cea7b977765a7", size = 228089, upload-time = "2026-10-30T15:10:00.308Z" }, + { url = "https://files.pythonhosted.org/packages/fe/7a/f523830e562da9b1d7a610174752753720c5e47e84b59b5fc47721d416bd/websockets-17.4-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:e0feb03a338f8f5e819ae23a73aabb7b46fd3b73f15f7df782dd12d99e2b02ad", size = 226743, upload-time = "2026-10-30T15:10:01.744Z" }, + { url = "https://files.pythonhosted.org/packages/2c/e6/d1a56495cb7b1a0a4688145131be38cfd0c1342bde8125e136a6e6ee494e/websockets-17.4-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:9ff89e36451a19775e7556d95ebfa4a0b24b84cc87565d6b5d0ec8ea82bf3f2c", size = 227237, upload-time = "2026-10-30T15:10:03.417Z" }, + { url = "https://files.pythonhosted.org/packages/44/41/c4e53f7ad63838d7eba6e82c5d743827e91dafe5e48c88a58f7e90043945/websockets-17.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:ee6c4e0085bff5f6621bf9c53452584dcb91ebd993ce841e1f4061ac12b6b556", size = 228348, upload-time = "2026-10-30T15:10:05.149Z" }, + { url = "https://files.pythonhosted.org/packages/ca/01/f6f78c3edeb8ad2b3c35a8c4117ec2cf9346215afaed8155af1e7933818b/websockets-17.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:1962f3bfde085af18878f1bb1aafb300c9a6ac33da886d0b0d97e33c9ce7a222", size = 226099, upload-time = "2026-10-30T15:10:06.588Z" }, + { url = "https://files.pythonhosted.org/packages/8d/01/d8c11740231b63d1aba6bad6b1d4e72e8384b26c5448801da238ebe82c92/websockets-17.4-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:42d7a6e818214336cf7af9a3aae5c6923f94f09fa07ac5aab98268ba2a98a6d5", size = 226956, upload-time = "2026-10-30T15:10:08.141Z" }, + { url = "https://files.pythonhosted.org/packages/99/87/dbfd43ae01ea8fdb7478a1b3d58b32aca83bb9c4aa61ebe187d1e188ac8e/websockets-17.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c9f8fdb7bbefc4907c2bb3593f810f2b67a2d5329e1d38724a1aa3d763cf63eb", size = 227253, upload-time = "2026-10-30T15:10:09.533Z" }, + { url = "https://files.pythonhosted.org/packages/bf/8e/fbd968a75428063560ada02d564de8d6c49ca34ec29dfb78e5d900f4b0ef/websockets-17.4-cp315-cp315t-win32.whl", hash = "sha256:506623128d7a55b726a5a46e4c9a8eae4c9dd0f04ff60b041fe7df85b9d034fe", size = 218877, upload-time = "2026-10-30T15:10:10.925Z" }, + { url = "https://files.pythonhosted.org/packages/f9/53/8f37a3943517bba77fc86f7c4162c455018c5b6d760c5e9c9a11de9e37f7/websockets-17.4-cp315-cp315t-win_amd64.whl", hash = "sha256:6e6558efbb256b532c9f06a21ba42bcac729718e365c505819f181f25e5c0eca", size = 219195, upload-time = "2026-10-30T15:10:12.336Z" }, + { url = "https://files.pythonhosted.org/packages/69/35/cf33cba7248fcd80cfe5911ffe67aa2637f50c2bfaada123b1e9d2aeca90/websockets-17.4-cp315-cp315t-win_arm64.whl", hash = "sha256:fcac44d917672ad6257ad64b80b5168306f168a78cab5c0612e4c699caed18cf", size = 219117, upload-time = "2026-10-30T15:10:13.771Z" }, + { url = "https://files.pythonhosted.org/packages/eb/27/9db924cb0dbf22eade78d98fbae77790f9bf6bc729e6ab29aef9d68e4156/websockets-17.4-py3-none-any.whl", hash = "sha256:c6bd2c9781915592b209160d0f017d4c7c82ad53075e6b0c24d2a89299e7f9e9", size = 212900, upload-time = "2026-10-30T15:10:34.482Z" }, +] + [[package]] name = "werkzeug" version = "3.1.9" From e3c190374f72d70140fea7634ec03e99924d78a9 Mon Sep 17 00:00:00 2001 From: ialarmedalien Date: Sat, 21 Nov 2026 08:38:20 -0800 Subject: [PATCH 2/2] test fix --- tests/cdm_data_loaders/core/test_transforms.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/cdm_data_loaders/core/test_transforms.py b/tests/cdm_data_loaders/core/test_transforms.py index f8e4ada2..77828a1d 100644 --- a/tests/cdm_data_loaders/core/test_transforms.py +++ b/tests/cdm_data_loaders/core/test_transforms.py @@ -1,6 +1,7 @@ """Tests for `cdm_data_loaders.core.transforms` and `TransformSettingsMixin`.""" import functools +import operator from collections.abc import Iterable from pathlib import Path from types import SimpleNamespace @@ -1458,8 +1459,8 @@ def test_annotation_capabilities_classifies_types(ann: object, expected_caps: tu pytest.param(Annotated[int, TransformContext], False, id="annotated_ctx_in_metadata_only"), pytest.param(TransformContext | None, True, id="union"), pytest.param(int | None, False, id="union_non_ctx"), - pytest.param(Union.__getitem__((TransformContext, None)), True, id="typing_union_ctx"), - pytest.param(Union.__getitem__((int, str)), False, id="typing_union_non_ctx"), + pytest.param(operator.getitem(Union, (TransformContext, None)), True, id="typing_union_ctx"), + pytest.param(operator.getitem(Union, (int, str)), False, id="typing_union_non_ctx"), pytest.param("TransformContext | None", True, id="str_ctx"), pytest.param("dict[str, Any]", False, id="str_non_ctx"), pytest.param(int, False, id="non_ctx"),