diff --git a/.gitignore b/.gitignore
index 92920303..9aa95f9f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
not_for_github/
cdl_skill/
+mutants/
.DS_Store
mutants/
diff --git a/pyproject.toml b/pyproject.toml
index 43914c81..cb029d92 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -83,6 +83,7 @@ models = [
]
mutants = [
"mutmut>=3.8.1",
+ "mutmut-mcp",
]
docs = [
"griffe-pydantic>=1.3.2",
@@ -287,4 +288,5 @@ DLT_PROJECT_DIR = "/tmp/dlt"
[tool.uv.sources]
cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" }
cdm-task-service-client = { git = "https://github.com/kbase/cdm-task-service-client.git" }
+mutmut-mcp = { git = "https://github.com/wdm0006/mutmut-mcp" }
# berdl-notebook-utils = { git = "https://github.com/KBaseDataLakehouse/spark_notebook.git", subdirectory = "notebook_utils" }
diff --git a/tests/cdm_data_loaders/cli/test_gz.py b/tests/cdm_data_loaders/cli/test_gz.py
index 8dafee2f..4e1bfe6b 100644
--- a/tests/cdm_data_loaders/cli/test_gz.py
+++ b/tests/cdm_data_loaders/cli/test_gz.py
@@ -69,6 +69,17 @@ def test_cli_directory_with_glob(tmp_path: Path, source_flag: str, glob_flag: st
assert not (sub_dir / "b.log.gz").exists()
+def test_main_directory_default_glob(tmp_path: Path) -> None:
+ """Compress all files in a directory using ``*`` when ``--file-glob`` is omitted."""
+ sub_dir = tmp_path / "sub"
+ sub_dir.mkdir()
+ for name in ("a.txt", "b.log"):
+ (sub_dir / name).write_text(name, encoding="utf-8")
+ main(["--source", str(sub_dir)])
+ assert (sub_dir / "a.txt.gz").exists()
+ assert (sub_dir / "b.log.gz").exists()
+
+
def test_cli_main_nonexistent_source(tmp_path: Path) -> None:
"""Raise ``RuntimeError`` when ``--source`` points to a missing path."""
missing = tmp_path / "doesnotexist"
diff --git a/tests/cdm_data_loaders/cli/test_ncbi_ftp_promote.py b/tests/cdm_data_loaders/cli/test_ncbi_ftp_promote.py
index 31c89b03..8420a41f 100644
--- a/tests/cdm_data_loaders/cli/test_ncbi_ftp_promote.py
+++ b/tests/cdm_data_loaders/cli/test_ncbi_ftp_promote.py
@@ -102,12 +102,6 @@
"dry_run": False,
}
-MOCK_REPORT_WITH_FAILURES: Final[dict[str, Any]] = {
- **MOCK_REPORT_SUCCESS,
- "promoted": 3,
- "failed": 2,
-}
-
def make_settings(**kwargs: str | int | bool | Path | PurePosixPath | None) -> PromoteSettings:
"""Build a validated ``PromoteSettings`` instance with a default ``staging_path``.
@@ -212,28 +206,31 @@ def test_run_promote_calls_promote_from_s3_with_correct_args(tmp_path: Path) ->
)
-def test_run_promote_no_error_on_zero_failures() -> None:
- """run_promote does not raise when promote_from_s3 reports zero failures."""
- config = make_settings()
- with patch(
- "cdm_data_loaders.cli.ncbi_ftp_promote.promote_from_s3",
- return_value=MOCK_REPORT_SUCCESS,
- ):
- run_promote(config)
-
-
-def test_run_promote_logs_error_on_failures(caplog: pytest.LogCaptureFixture) -> None:
- """run_promote logs an error without raising when promote_from_s3 reports failures."""
+@pytest.mark.parametrize(
+ ("failed_count", "expected_messages"),
+ [
+ pytest.param(0, [], id="zero-failures"),
+ pytest.param(1, ["Promote completed with 1 failures"], id="single-failure"),
+ pytest.param(2, ["Promote completed with 2 failures"], id="multiple-failures"),
+ ],
+)
+def test_run_promote_logs_error_only_when_failures_positive(
+ caplog: pytest.LogCaptureFixture,
+ failed_count: int,
+ expected_messages: list[str],
+) -> None:
+ """run_promote logs an error without raising if and only if promote_from_s3 reports failures > 0."""
config = make_settings()
+ report = {**MOCK_REPORT_SUCCESS, "failed": failed_count}
with (
patch(
"cdm_data_loaders.cli.ncbi_ftp_promote.promote_from_s3",
- return_value=MOCK_REPORT_WITH_FAILURES,
+ return_value=report,
),
- caplog.at_level(logging.ERROR),
+ caplog.at_level(logging.ERROR, logger="cdm_data_loaders.cli.ncbi_ftp_promote"),
):
run_promote(config)
- assert "Promote completed with 2 failures" in caplog.text
+ assert caplog.messages == expected_messages
def test_run_promote_dry_run_forwarded() -> None:
diff --git a/tests/cdm_data_loaders/cli/test_ncbi_ftp_sync.py b/tests/cdm_data_loaders/cli/test_ncbi_ftp_sync.py
index dbbce73a..a0d0e6b6 100644
--- a/tests/cdm_data_loaders/cli/test_ncbi_ftp_sync.py
+++ b/tests/cdm_data_loaders/cli/test_ncbi_ftp_sync.py
@@ -1,6 +1,8 @@
"""Tests for cli.ncbi_ftp_sync — settings and CLI shortcuts."""
+import logging
from pathlib import Path
+from unittest.mock import patch
import pytest
from pydantic import ValidationError
@@ -8,6 +10,7 @@
from cdm_data_loaders.cli.ncbi_ftp_sync import (
DownloadSettings,
+ run_download,
)
from cdm_data_loaders.core.fields import INPUT_MOUNT, OUTPUT_MOUNT
from cdm_data_loaders.datasets.ncbi_ftp.constants import FTP_HOST
@@ -105,3 +108,46 @@ def test_settings_threads_boundaries_accepted(threads: int) -> None:
"""Boundary thread counts (1 and 32) are accepted."""
s = DownloadSettings(threads=threads) # pyright: ignore[reportCallIssue]
assert s.threads == threads
+
+
+@pytest.mark.parametrize(
+ ("failed_count", "expected_messages"),
+ [
+ pytest.param(0, [], id="no-failures"),
+ pytest.param(1, ["Download completed with 1 failures"], id="single-failure"),
+ pytest.param(2, ["Download completed with 2 failures"], id="multiple-failures"),
+ ],
+)
+def test_run_download_forwards_settings_and_logs_failures(
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+ failed_count: int,
+ expected_messages: list[str],
+) -> None:
+ """run_download forwards all DownloadSettings fields to download_batch and logs failures > 0."""
+ manifest = tmp_path / "manifest.txt"
+ out_dir = tmp_path / "out"
+ settings = DownloadSettings(
+ manifest=manifest,
+ output_dir=out_dir,
+ threads=CUSTOM_THREADS,
+ ftp_host="ftp.example.org",
+ limit=CUSTOM_LIMIT,
+ )
+ with (
+ patch(
+ "cdm_data_loaders.cli.ncbi_ftp_sync.download_batch",
+ return_value={"failed": failed_count},
+ ) as mock_batch,
+ caplog.at_level(logging.ERROR, logger="cdm_data_loaders.cli.ncbi_ftp_sync"),
+ ):
+ run_download(settings)
+
+ mock_batch.assert_called_once_with(
+ manifest_path=manifest,
+ output_dir=out_dir,
+ threads=CUSTOM_THREADS,
+ ftp_host="ftp.example.org",
+ limit=CUSTOM_LIMIT,
+ )
+ assert caplog.messages == expected_messages
diff --git a/tests/cdm_data_loaders/cli/test_polaris_schema_definitions.py b/tests/cdm_data_loaders/cli/test_polaris_schema_definitions.py
index bd5458cc..d98a57db 100644
--- a/tests/cdm_data_loaders/cli/test_polaris_schema_definitions.py
+++ b/tests/cdm_data_loaders/cli/test_polaris_schema_definitions.py
@@ -1,9 +1,12 @@
"""Unit and integration tests for cdm_data_loaders.cli.polaris_schema_definitions."""
import json
+import logging
from collections.abc import Generator
+from contextlib import contextmanager
from pathlib import Path
from typing import Any, Final
+from unittest.mock import patch
import dlt
import pyarrow as pa
@@ -16,7 +19,9 @@
from pyiceberg.utils.config import Config
from cdm_data_loaders.cli.polaris_schema_definitions import (
+ PolarisSchemaDefinitionsSettings,
load_pyarrow_schema,
+ run_from_settings,
)
from cdm_data_loaders.cli.polaris_schema_definitions import (
main as cli_main,
@@ -722,3 +727,118 @@ def test_cli_main_applies_definitions_and_prints_json_report(
updated_tbl = verify_cat.load_table((NAMESPACE, "id_mapping"))
assert updated_tbl.properties["comment"] == "Standardised mapping data from UniProt"
assert collect_iceberg_docs(updated_tbl.schema()) == parsed_stdout["updated_fields"]
+
+
+def test_load_pyarrow_schema_blank_description_returns_none(tmp_path: Path) -> None:
+ """load_pyarrow_schema produces no metadata definition for whitespace-only descriptions."""
+ json_path = tmp_path / "blank_desc.json"
+ json_path.write_text(
+ json.dumps(
+ {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "type": "object",
+ "description": " ",
+ "properties": {"id": {"type": "string"}},
+ }
+ ),
+ encoding="utf-8",
+ )
+ loaded = load_pyarrow_schema(str(json_path))
+ assert extract_metadata_definition(loaded.metadata) is None
+
+
+@pytest.mark.parametrize(
+ ("dry_run", "expected_verb"),
+ [
+ pytest.param(True, "Planned", id="dry-run"),
+ pytest.param(False, "Applied", id="apply"),
+ ],
+)
+def test_run_from_settings_forwards_all_options_and_logs(
+ tmp_path: Path,
+ caplog: pytest.LogCaptureFixture,
+ *,
+ dry_run: bool,
+ expected_verb: str,
+) -> None:
+ """run_from_settings forwards catalog and sync options and logs Planned or Applied."""
+ json_path = tmp_path / "schema.json"
+ json_path.write_text(
+ json.dumps(
+ {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "type": "object",
+ "properties": {"id": {"type": "string"}},
+ }
+ ),
+ encoding="utf-8",
+ )
+
+ captured_cfg: list[PolarisCatalogConfig] = []
+ captured_opts: list[DefinitionSyncOptions] = []
+
+ @contextmanager
+ def fake_connect(cfg: PolarisCatalogConfig) -> Generator[object]:
+ captured_cfg.append(cfg)
+ yield object()
+
+ def fake_apply(
+ _cat: object,
+ _table: str,
+ _schema: pa.Schema,
+ *,
+ options: DefinitionSyncOptions,
+ ) -> SchemaDefinitionsReport:
+ captured_opts.append(options)
+ return SchemaDefinitionsReport(
+ table_identifier=("ns", "tbl"),
+ dry_run=options.dry_run,
+ table_comment_applied="desc",
+ updated_fields={"a": "b"},
+ unchanged_fields=["c"],
+ )
+
+ bearer_val = "tok"
+ settings = PolarisSchemaDefinitionsSettings(
+ catalog="mycat",
+ table="ns.tbl",
+ schema_source=str(json_path),
+ table_description="override desc",
+ uri="http://polaris:8181/api/catalog",
+ warehouse="wh",
+ credential="cid:csec",
+ token=bearer_val,
+ scope="ROLE:ALL",
+ metadata_keys=("doc",),
+ normalize_root_names=False,
+ strict=True,
+ dry_run=dry_run,
+ )
+ with (
+ patch("cdm_data_loaders.cli.polaris_schema_definitions.connect_polaris_catalog", fake_connect),
+ patch("cdm_data_loaders.cli.polaris_schema_definitions.apply_schema_definitions", fake_apply),
+ caplog.at_level(logging.INFO, logger="cdm_data_loaders.cli.polaris_schema_definitions"),
+ ):
+ report = run_from_settings(settings)
+
+ assert report.dry_run is dry_run
+ assert captured_cfg == [
+ PolarisCatalogConfig(
+ catalog="mycat",
+ uri="http://polaris:8181/api/catalog",
+ warehouse="wh",
+ credential="cid:csec",
+ token=bearer_val,
+ scope="ROLE:ALL",
+ )
+ ]
+ assert captured_opts == [
+ DefinitionSyncOptions(
+ table_description="override desc",
+ metadata_keys=("doc",),
+ normalize_root_names=False,
+ strict=True,
+ dry_run=dry_run,
+ )
+ ]
+ assert f"{expected_verb} definitions on ns.tbl" in caplog.text
diff --git a/tests/cdm_data_loaders/cli/test_pyiceberg_to_jsonschema.py b/tests/cdm_data_loaders/cli/test_pyiceberg_to_jsonschema.py
index c52e7d30..9ce29d86 100644
--- a/tests/cdm_data_loaders/cli/test_pyiceberg_to_jsonschema.py
+++ b/tests/cdm_data_loaders/cli/test_pyiceberg_to_jsonschema.py
@@ -17,6 +17,7 @@
from linkml_runtime.linkml_model.meta import SchemaDefinition
from linkml_runtime.loaders import yaml_loader
from pyiceberg.catalog import Catalog
+from pyiceberg.catalog.sql import SqlCatalog
from pyiceberg.schema import Schema
from pyiceberg.types import BinaryType, NestedField
@@ -47,17 +48,17 @@ class CatalogDumpTarget:
settings_cls: type[IcebergCatalogDumpSettings]
dump_fn: Callable[[Any], None]
suffix: str
- create_bad_table: Callable[[Catalog, Schema], None]
+ create_bad_table: Callable[[Catalog, Schema, str], None]
assert_table_written: Callable[[Path, str, bool], None]
-def _create_bad_jsonschema_table(catalog: Catalog, _simple_schema: Schema) -> None:
+def _create_bad_jsonschema_table(catalog: Catalog, _simple_schema: Schema, table_name: str) -> None:
unsupported_schema = Schema(NestedField(1, "value", BinaryType(), required=True))
- catalog.create_table(("ns", "bad"), schema=unsupported_schema)
+ catalog.create_table(("ns", table_name), schema=unsupported_schema)
-def _create_bad_linkml_table(catalog: Catalog, simple_schema: Schema) -> None:
- bad_table = catalog.create_table(("ns", "bad"), schema=simple_schema)
+def _create_bad_linkml_table(catalog: Catalog, simple_schema: Schema, table_name: str) -> None:
+ bad_table = catalog.create_table(("ns", table_name), schema=simple_schema)
bad_table.io.delete(bad_table.metadata_location)
@@ -98,7 +99,12 @@ def _assert_linkml_written(path: Path, table_name: str, group_by_namespace: bool
@pytest.mark.parametrize("group_by_namespace", [False, True], ids=["per-table", "per-namespace"])
def test_end_to_end_dump_catalog_schemas_writes_valid_docs(
- catalog: Catalog, catalog_env: Path, simple_schema: Schema, wide_schema: Schema, group_by_namespace: bool
+ catalog: Catalog,
+ catalog_env: Path,
+ simple_schema: Schema,
+ wide_schema: Schema,
+ caplog: pytest.LogCaptureFixture,
+ group_by_namespace: bool,
) -> None:
"""dump_catalog_schemas loads the catalog by name and writes valid schema docs, per table or per namespace."""
catalog.create_namespace("ns")
@@ -109,12 +115,18 @@ def test_end_to_end_dump_catalog_schemas_writes_valid_docs(
settings = IcebergToJsonSchemaSettings( # pyright: ignore[reportCallIssue]
catalog=CATALOG_NAME, output_dir=str(out_dir), group_by_namespace=group_by_namespace
)
- dump_catalog_schemas(settings)
+ with caplog.at_level(logging.INFO, logger="cdm_data_loaders.cli.pyiceberg_to_jsonschema"):
+ dump_catalog_schemas(settings)
+
+ assert caplog.messages[0] == f"Dumping schemas for 1 namespace(s) from catalog {CATALOG_NAME!r}"
files = sorted(out_dir.iterdir())
assert [f.name for f in files] == (
["ns.schema.json"] if group_by_namespace else ["ns.simple.schema.json", "ns.wide.schema.json"]
)
+ for file_path in files:
+ raw_text = file_path.read_text(encoding="utf-8")
+ assert raw_text == json.dumps(json.loads(raw_text), indent=2)
if group_by_namespace:
namespace_doc = json.loads(files[0].read_text())
@@ -205,9 +217,10 @@ def test_dump_catalog_schemas_pass_skips_failing_table_and_logs(
group_by_namespace: bool,
include_good: bool,
) -> None:
- """A table that fails to convert is logged and skipped; other tables still succeed."""
+ """Tables that fail to convert are logged, counted, and skipped; other tables still succeed."""
catalog.create_namespace("ns")
- target.create_bad_table(catalog, simple_schema)
+ target.create_bad_table(catalog, simple_schema, "bad_a")
+ target.create_bad_table(catalog, simple_schema, "bad_b")
if include_good:
catalog.create_table(("ns", "good"), schema=simple_schema)
@@ -215,15 +228,18 @@ def test_dump_catalog_schemas_pass_skips_failing_table_and_logs(
settings = target.settings_cls( # pyright: ignore[reportCallIssue]
catalog=CATALOG_NAME, output_dir=str(out_dir), group_by_namespace=group_by_namespace
)
- with caplog.at_level(logging.WARNING):
+ with caplog.at_level(logging.WARNING, logger=target.dump_fn.__module__):
target.dump_fn(settings)
expected_files = [f"ns{target.suffix}" if group_by_namespace else f"ns.good{target.suffix}"] if include_good else []
assert [f.name for f in sorted(out_dir.iterdir())] == expected_files
if include_good:
target.assert_table_written(out_dir / expected_files[0], "good", group_by_namespace)
- assert "Failed to convert table ('ns', 'bad'); skipping" in caplog.text
- assert "Completed with 1 table(s) skipped due to conversion failures" in caplog.text
+ assert caplog.messages == [
+ "Failed to convert table ('ns', 'bad_a'); skipping",
+ "Failed to convert table ('ns', 'bad_b'); skipping",
+ "Completed with 2 table(s) skipped due to conversion failures",
+ ]
@pytest.mark.parametrize("target", CATALOG_DUMP_TARGETS)
@@ -283,6 +299,27 @@ def test_dump_catalog_schemas_pass_namespace_isolation(
assert first["x-iceberg"]["generated_at"] == second["x-iceberg"]["generated_at"]
+def test_dump_catalog_schemas_pass_multilevel_namespace(
+ catalog: Catalog, catalog_env: Path, simple_schema: Schema, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Multi-level namespace tuples are dot-joined for grouped filenames and schema metadata."""
+ catalog.create_namespace(("parent", "child"))
+ catalog.create_table(("parent", "child", "record"), schema=simple_schema)
+ monkeypatch.setattr(SqlCatalog, "list_namespaces", lambda _self, _namespace=(): [("parent", "child")])
+
+ out_dir = catalog_env / "schemas"
+ settings = IcebergToJsonSchemaSettings( # pyright: ignore[reportCallIssue]
+ catalog=CATALOG_NAME, output_dir=str(out_dir), group_by_namespace=True
+ )
+ dump_catalog_schemas(settings)
+
+ assert [file.name for file in out_dir.iterdir()] == ["parent.child.schema.json"]
+ document = json.loads((out_dir / "parent.child.schema.json").read_text(encoding="utf-8"))
+ assert document["$id"] == "urn:iceberg:parent.child"
+ assert document["title"] == "parent.child"
+ assert set(document["$defs"]) == {"record"}
+
+
@pytest.mark.parametrize("target", CATALOG_DUMP_TARGETS)
@pytest.mark.parametrize(
("arguments", "expected"),
diff --git a/tests/cdm_data_loaders/coercion/test_xml.py b/tests/cdm_data_loaders/coercion/test_xml.py
index dc7baf26..d0150acd 100644
--- a/tests/cdm_data_loaders/coercion/test_xml.py
+++ b/tests/cdm_data_loaders/coercion/test_xml.py
@@ -1,8 +1,9 @@
"""Unit tests for XML-to-Arrow record coercion helpers."""
+import datetime as dt
import functools
from pathlib import Path
-from typing import Final
+from typing import Any, Final
import pyarrow as pa
import pytest
@@ -14,7 +15,10 @@
make_xml_element_converter,
make_xml_level_converters,
)
-from cdm_data_loaders.converters.exceptions.facades.xsd_to_pyarrow import WildcardAttributeError
+from cdm_data_loaders.converters.exceptions.facades.xsd_to_pyarrow import (
+ FieldNameCollisionError,
+ WildcardAttributeError,
+)
from cdm_data_loaders.schemas.uniprot.schema import SCHEMAS as UNIPROT_SCHEMAS
from cdm_data_loaders.validation.xsd.traversal import load_schema
from tests.helpers.paths_and_constants import BASE_DIR
@@ -106,3 +110,172 @@ def test_xml_coercion_entry_points_fail_invalid_max_recursion_depth(
with pytest.raises(exc_cls, match="max_recursion_depth"):
call()
+
+
+COERCION_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+
+@pytest.fixture
+def coercion_schema() -> xmlschema.XMLSchema:
+ """Compile the XSD schema fixture used by convert_xml_element tests."""
+ return xmlschema.XMLSchema(COERCION_XSD)
+
+
+@pytest.mark.parametrize(
+ ("xml_bytes", "xml_tag", "max_recursion_depth", "expected"),
+ [
+ pytest.param(
+ b"0a1bYWJj2026-01-02Z",
+ "scalars",
+ None,
+ {"hex_val": b"\x0a\x1b", "b64_val": b"abc", "date_val": dt.date(2026, 1, 2)},
+ id="scalars",
+ ),
+ pytest.param(
+ (
+ b''
+ b""
+ b''
+ b"a b"
+ b"1 2 3"
+ b'hi <b> bold tail
'
+ b'42'
+ b''
+ b''
+ b''
+ b'x'
+ b""
+ ),
+ "root",
+ 1,
+ {
+ "_kind": "top",
+ "req_str": "",
+ "nmtokens": ["a", "b"],
+ "int_list": [1, 2, 3],
+ "mixed_attr": {
+ "_id": "m1",
+ "#text": 'hi <b> bold tail
',
+ },
+ "sc": {"_unit": "kg", "#text": 42},
+ "empty_attr": {"_flag": True},
+ "child": [None, '{"_kind": "leaf", "req_str": "x"}'],
+ },
+ id="mixed-nil-lists-and-truncated-child",
+ ),
+ pytest.param(
+ b'',
+ "root",
+ None,
+ {"_kind": "k"},
+ id="nil-root-with-attribute",
+ ),
+ pytest.param(
+ b'',
+ "root",
+ None,
+ {},
+ id="nil-root-empty",
+ ),
+ ],
+)
+def test_convert_xml_element_pass_scalars_mixed_nil_and_lists(
+ coercion_schema: xmlschema.XMLSchema,
+ xml_bytes: bytes,
+ xml_tag: str,
+ max_recursion_depth: int | None,
+ expected: dict[str, Any],
+) -> None:
+ """convert_xml_element coerces scalars, xsi:nil elements, mixed inline XML, lists, and truncated recursion."""
+ result = (
+ convert_xml_element(fromstring(xml_bytes), coercion_schema, xml_tag)
+ if max_recursion_depth is None
+ else convert_xml_element(
+ fromstring(xml_bytes),
+ coercion_schema,
+ xml_tag,
+ max_recursion_depth=max_recursion_depth,
+ )
+ )
+ assert result == expected
+
+
+def test_convert_xml_element_fail_attribute_child_collision() -> None:
+ """convert_xml_element raises FieldNameCollisionError when an attribute and child map to the same name."""
+ collision_xsd = xmlschema.XMLSchema(
+ """
+
+
+
+
+
+
+
+
+ """
+ )
+ with pytest.raises(FieldNameCollisionError, match="Field name collision"):
+ convert_xml_element(fromstring(b'<_id>2'), collision_xsd, "root")
+
+
+def test_make_xml_level_converters_pass_nil_leaf_element() -> None:
+ """make_xml_level_converters returns an empty dict for a nil scalar leaf element."""
+ leaf_xsd = xmlschema.XMLSchema(
+ """
+
+
+
+
+
+
+
+ """
+ )
+ note_conv = make_xml_level_converters(leaf_xsd, 1)["note"]
+ assert note_conv(fromstring(b'')) == {}
diff --git a/tests/cdm_data_loaders/converters/core/test_inference.py b/tests/cdm_data_loaders/converters/core/test_inference.py
index 0a580044..7bf0cc91 100644
--- a/tests/cdm_data_loaders/converters/core/test_inference.py
+++ b/tests/cdm_data_loaders/converters/core/test_inference.py
@@ -1,7 +1,7 @@
"""Tests for target-independent type inference."""
from decimal import Decimal
-from typing import Any
+from typing import Any, Final
import pytest
@@ -15,8 +15,53 @@
infer_implicit_type,
is_unconstrained_node,
json_type_from_enum,
+ nodes_same_shape,
+ unify_scalar_specs,
+)
+from cdm_data_loaders.converters.core.ir import (
+ SPEC_FLOAT64,
+ SPEC_STRING,
+ SPEC_TS_UTC,
+ Field,
+ NodeHints,
+ TypedNode,
)
-from cdm_data_loaders.converters.core.ir import TypedNode
+
+DEC_10_2: Final[tuple[str, NodeHints, str]] = (
+ "number",
+ NodeHints(logical_type="decimal", precision=10, scale=2),
+ "Decimal(10, 2)",
+)
+DEC_12_4: Final[tuple[str, NodeHints, str]] = (
+ "number",
+ NodeHints(logical_type="decimal", precision=12, scale=4),
+ "Decimal(12, 4)",
+)
+DEC_38_0: Final[tuple[str, NodeHints, str]] = (
+ "number",
+ NodeHints(logical_type="decimal", precision=38, scale=0),
+ "Decimal(38, 0)",
+)
+LONG_SPEC: Final[tuple[str, NodeHints, str]] = ("integer", NodeHints(logical_type="long", bit_width=64), "int")
+TS_TZ_SPEC: Final[tuple[str, NodeHints, str]] = (
+ "string",
+ NodeHints(logical_type="timestamp-with-tz", timezone=True),
+ "AwareDT",
+)
+TS_NAIVE_SPEC: Final[tuple[str, NodeHints, str]] = (
+ "string",
+ NodeHints(logical_type="timestamp-without-tz", timezone=False),
+ "NaiveDT",
+)
+BOOL_SPEC: Final[tuple[str, NodeHints, str]] = ("boolean", NodeHints(logical_type="boolean"), "bool")
+
+STR_NODE: Final[TypedNode] = TypedNode("string")
+NULLABLE_STR_NODE: Final[TypedNode] = TypedNode("string", nullable=True)
+INT_NODE: Final[TypedNode] = TypedNode("integer")
+DATE_STR_NODE: Final[TypedNode] = TypedNode("string", hints=NodeHints(logical_type="date"))
+ARRAY_STR_NODE: Final[TypedNode] = TypedNode("array", items=STR_NODE)
+MAP_STR_INT_NODE: Final[TypedNode] = TypedNode("map", key_type=STR_NODE, value_type=INT_NODE)
+OBJ_X_REQ_NODE: Final[TypedNode] = TypedNode("object", properties=(Field("x", STR_NODE, required=True),))
@pytest.mark.parametrize(
@@ -178,11 +223,119 @@ def test_decimal_places_pass_scale(value: float | Decimal, expected: int) -> Non
("precision", "scale", "expected"),
[
(10, 2, r"^-?\d{1,8}(\.\d{1,2})?$"),
+ (5, 2, r"^-?\d{1,3}(\.\d{1,2})?$"),
(2, 2, r"^-?0(\.\d{1,2})?$"),
(10, 0, r"^-?\d{1,10}$"),
+ (4, 0, r"^-?\d{1,4}$"),
],
- ids=["standard", "zero-before-decimal", "no-scale"],
+ ids=["standard", "small-standard", "zero-before-decimal", "no-scale", "small-no-scale"],
)
def test_decimal_pattern_pass_regex(precision: int, scale: int, expected: str) -> None:
"""Build precise regex patterns for decimal constraints."""
assert decimal_pattern(precision, scale) == expected
+
+
+@pytest.mark.parametrize(
+ ("specs", "expected"),
+ [
+ pytest.param([DEC_10_2, DEC_10_2], ("number", DEC_10_2[1], "Decimal(10, 2)"), id="identical-decimals"),
+ pytest.param(
+ [DEC_10_2, DEC_12_4],
+ ("number", NodeHints(logical_type="decimal", precision=12, scale=4), "Decimal(10, 2) | Decimal(12, 4)"),
+ id="widened-decimals",
+ ),
+ pytest.param(
+ [DEC_38_0, DEC_12_4],
+ (*SPEC_FLOAT64, "Decimal(38, 0) | Decimal(12, 4)"),
+ id="decimal-precision-overflow",
+ ),
+ pytest.param([LONG_SPEC, DEC_10_2], (*SPEC_FLOAT64, "int | Decimal(10, 2)"), id="integer-and-decimal"),
+ pytest.param([TS_TZ_SPEC, TS_NAIVE_SPEC], (*SPEC_TS_UTC, "AwareDT | NaiveDT"), id="mixed-timestamps"),
+ pytest.param([LONG_SPEC, BOOL_SPEC], (*SPEC_STRING, "int | bool"), id="incompatible-fallback"),
+ ],
+)
+def test_unify_scalar_specs_pass(
+ specs: list[tuple[str, NodeHints, str]],
+ expected: tuple[str, NodeHints, str],
+) -> None:
+ """unify_scalar_specs unifies identical specs, decimals, numeric or timestamp groups, and fallbacks."""
+ assert unify_scalar_specs(specs) == expected
+
+
+@pytest.mark.parametrize(
+ ("left", "right", "expected"),
+ [
+ pytest.param(STR_NODE, NULLABLE_STR_NODE, True, id="scalar-ignores-nullability"),
+ pytest.param(STR_NODE, INT_NODE, False, id="scalar-different-type"),
+ pytest.param(STR_NODE, DATE_STR_NODE, False, id="scalar-different-hints"),
+ pytest.param(ARRAY_STR_NODE, TypedNode("array", items=STR_NODE), True, id="array-same-items"),
+ pytest.param(ARRAY_STR_NODE, TypedNode("array", items=NULLABLE_STR_NODE), False, id="array-item-nullability"),
+ pytest.param(ARRAY_STR_NODE, TypedNode("array", items=INT_NODE), False, id="array-item-type"),
+ pytest.param(ARRAY_STR_NODE, TypedNode("array", items=None), False, id="array-missing-items"),
+ pytest.param(ARRAY_STR_NODE, TypedNode("array", items=(STR_NODE,)), False, id="array-tuple-items"),
+ pytest.param(
+ MAP_STR_INT_NODE,
+ TypedNode("map", key_type=STR_NODE, value_type=INT_NODE),
+ True,
+ id="map-same-types",
+ ),
+ pytest.param(
+ MAP_STR_INT_NODE,
+ TypedNode("map", key_type=None, value_type=INT_NODE),
+ False,
+ id="map-missing-key",
+ ),
+ pytest.param(
+ MAP_STR_INT_NODE,
+ TypedNode("map", key_type=STR_NODE, value_type=None),
+ False,
+ id="map-missing-value",
+ ),
+ pytest.param(
+ MAP_STR_INT_NODE,
+ TypedNode("map", key_type=INT_NODE, value_type=INT_NODE),
+ False,
+ id="map-different-key",
+ ),
+ pytest.param(
+ MAP_STR_INT_NODE,
+ TypedNode("map", key_type=STR_NODE, value_type=STR_NODE),
+ False,
+ id="map-different-value",
+ ),
+ pytest.param(
+ MAP_STR_INT_NODE,
+ TypedNode("map", key_type=STR_NODE, value_type=TypedNode("integer", nullable=True)),
+ False,
+ id="map-value-nullability",
+ ),
+ pytest.param(
+ OBJ_X_REQ_NODE,
+ TypedNode("object", properties=(Field("x", STR_NODE, required=True),)),
+ True,
+ id="object-same-properties",
+ ),
+ pytest.param(OBJ_X_REQ_NODE, TypedNode("object", properties=()), False, id="object-property-count"),
+ pytest.param(
+ OBJ_X_REQ_NODE,
+ TypedNode("object", properties=(Field("y", STR_NODE, required=True),)),
+ False,
+ id="object-property-name",
+ ),
+ pytest.param(
+ OBJ_X_REQ_NODE,
+ TypedNode("object", properties=(Field("x", STR_NODE, required=False),)),
+ False,
+ id="object-property-required",
+ ),
+ pytest.param(
+ OBJ_X_REQ_NODE,
+ TypedNode("object", properties=(Field("x", INT_NODE, required=True),)),
+ False,
+ id="object-property-type",
+ ),
+ ],
+)
+def test_nodes_same_shape_pass(left: TypedNode, right: TypedNode, *, expected: bool) -> None:
+ """nodes_same_shape compares types, hints, arrays, maps, and objects recursively."""
+ assert nodes_same_shape(left, right) is expected
diff --git a/tests/cdm_data_loaders/converters/emitters/test_dlt.py b/tests/cdm_data_loaders/converters/emitters/test_dlt.py
index f62ef710..e17dfcfd 100644
--- a/tests/cdm_data_loaders/converters/emitters/test_dlt.py
+++ b/tests/cdm_data_loaders/converters/emitters/test_dlt.py
@@ -539,3 +539,103 @@ def test_emit_pass_native_empty_table_and_new_child(
new_child = Field("new", TypedNode("object", properties=(Field("value", TypedNode("string")),)))
document = replace(document, root=replace(document.root, properties=(new_child,)))
assert DltEmitter(schema_name="source").emit(document)["tables"] == expected_tables
+
+
+def test_emit_pass_json_unions_and_boolean_warnings(caplog: pytest.LogCaptureFixture) -> None:
+ """DltEmitter approximates multi-type unions, anyOf/oneOf, and boolean schemas with warnings."""
+ source = base_object_schema(
+ properties={
+ "null_only": {"type": ["null"]},
+ "multi_union": {"type": ["string", "integer"]},
+ "nullable_str": {"type": ["string", "null"]},
+ "anyof_bool": {"anyOf": [True, {"type": "integer"}]},
+ "oneof_typed": {"oneOf": [{"type": "integer"}, {"type": "string"}]},
+ "bool_false": False,
+ },
+ )
+ doc = JsonSchemaReader().read(source)
+ with caplog.at_level("WARNING", logger="cdm_data_loaders.converters.emitters.dlt"):
+ tables = DltEmitter(schema_name="u").emit(doc)["tables"]
+ cols = tables["u"]["columns"]
+ assert cols["null_only"]["data_type"] == "text"
+ assert cols["multi_union"]["data_type"] == "text"
+ assert cols["nullable_str"]["data_type"] == "text"
+ assert cols["anyof_bool"]["data_type"] == "text"
+ assert cols["oneof_typed"]["data_type"] == "bigint"
+ assert cols["bool_false"]["data_type"] == "text"
+
+ messages = "\n".join(r.getMessage() for r in caplog.records)
+ assert "Collapsing multi-type union" in messages
+ assert "Approximating 'anyOf'" in messages
+ assert "Approximating 'oneOf'" in messages
+ assert "Boolean schema for property 'bool_false'" in messages
+
+
+def test_emit_pass_column_hints_precision_and_timezone() -> None:
+ """DltEmitter emits precision and timezone column hints for timestamps, fixed lengths, and bit widths."""
+ ts_tz = TypedNode("string", hints=NodeHints(logical_type="timestamp-with-tz", timezone=True))
+ ts_no_tz = TypedNode("string", hints=NodeHints(logical_type="timestamp-without-tz", timezone=False))
+ fixed_len = TypedNode("string", hints=NodeHints(logical_type="fixed", length=16))
+ int_bits = TypedNode("integer", hints=NodeHints(bit_width=32))
+ hints_doc = SchemaDocument(
+ root=TypedNode(
+ "object",
+ properties=(
+ Field("ts_tz", ts_tz),
+ Field("ts_no_tz", ts_no_tz),
+ Field("fixed_len", fixed_len),
+ Field("int_bits", int_bits),
+ ),
+ )
+ )
+ hint_cols = DltEmitter(schema_name="h").emit(hints_doc)["tables"]["h"]["columns"]
+ ts_precision = 6
+ fixed_precision = 16
+ int_precision = 32
+ assert hint_cols["ts_tz"]["precision"] == ts_precision
+ assert hint_cols["ts_tz"]["timezone"] is True
+ assert hint_cols["ts_no_tz"]["precision"] == ts_precision
+ assert hint_cols["ts_no_tz"]["timezone"] is False
+ assert hint_cols["fixed_len"]["precision"] == fixed_precision
+ assert hint_cols["int_bits"]["precision"] == int_precision
+
+
+def test_emit_pass_native_restore_table_and_column_metadata() -> None:
+ """Native dlt provenance restores table and column metadata, nullable/precision None values, and descriptions."""
+ source = {
+ "root": {
+ "name": "root",
+ "description": "old root desc",
+ "columns": {
+ "c_named": {"name": "c_named", "data_type": "text", "nullable": True, "precision": None},
+ "c_any": {"data_type": "text", "nullable": False, "scale": None, "timezone": None},
+ "c_null_none": {"data_type": "bigint", "nullable": False},
+ },
+ },
+ "root__sub": {"name": "root__sub", "parent": "root", "description": "sub desc", "columns": {}},
+ }
+ doc = DltReader().read(source)["root"]
+ # drop root description and override field nullability/type to exercise native metadata restoration
+ root_props = list(doc.root.properties)
+ c_named, c_any, c_null_none, sub_field = root_props
+ c_any_mod = replace(c_any, node=TypedNode("any", nullable=True, extensions=c_any.node.extensions))
+ c_null_none_mod = replace(c_null_none, node=replace(c_null_none.node, nullable=None))
+ doc_mod = replace(
+ doc,
+ root=replace(
+ doc.root,
+ annotations={},
+ properties=(c_named, c_any_mod, c_null_none_mod, sub_field),
+ ),
+ )
+ tables = DltEmitter(schema_name="s").emit(doc_mod)["tables"]
+ assert "description" not in tables["root"]
+ assert tables["root"]["name"] == "root"
+ assert tables["root"]["columns"]["c_named"]["name"] == "c_named"
+ assert tables["root"]["columns"]["c_named"]["precision"] is None
+ assert tables["root"]["columns"]["c_any"]["data_type"] is None
+ assert tables["root"]["columns"]["c_any"]["scale"] is None
+ assert tables["root"]["columns"]["c_any"]["timezone"] is None
+ assert tables["root"]["columns"]["c_null_none"]["nullable"] is None
+ assert tables["root__sub"]["description"] == "sub desc"
+ assert tables["root__sub"]["columns"] == {}
diff --git a/tests/cdm_data_loaders/converters/emitters/test_linkml.py b/tests/cdm_data_loaders/converters/emitters/test_linkml.py
index 83fc0698..6778f94f 100644
--- a/tests/cdm_data_loaders/converters/emitters/test_linkml.py
+++ b/tests/cdm_data_loaders/converters/emitters/test_linkml.py
@@ -8,9 +8,13 @@
from linkml_runtime.linkml_model.meta import ClassDefinition, EnumDefinition, SchemaDefinition, SlotDefinition
from linkml_runtime.loaders import yaml_loader
-from cdm_data_loaders.converters.core.ir import Field, SchemaDocument, TypedNode
+from cdm_data_loaders.converters.core.ir import Field, NodeHints, Provenance, SchemaDocument, TypedNode
from cdm_data_loaders.converters.emitters.linkml import LinkMLEmitter
-from cdm_data_loaders.converters.exceptions.emitters.linkml import LinkMLEmitterError
+from cdm_data_loaders.converters.exceptions.emitters.linkml import (
+ EmptyLinkMLSlotNameError,
+ LinkMLEmitterError,
+ NestedLinkMLMultivaluedError,
+)
from cdm_data_loaders.converters.sources.json_schema.permissive import JsonSchemaReader
@@ -262,3 +266,116 @@ def test_emit_into_fail_preserves_destination(root: TypedNode) -> None:
"good": ClassDefinition(name="good"),
}
assert schema.enums == {}
+
+
+def test_emit_field_fail_blank_name() -> None:
+ """emit_field rejects whitespace-only slot names with EmptyLinkMLSlotNameError."""
+ with pytest.raises(EmptyLinkMLSlotNameError, match="non-empty"):
+ LinkMLEmitter().emit_field(Field(" ", TypedNode("string")))
+
+
+def test_emit_field_pass_node_description() -> None:
+ """emit_field falls back to the node description when the field has no annotation."""
+ slot = LinkMLEmitter().emit_field(Field("col", TypedNode("string", annotations={"description": "from node"})))
+ assert slot.description == "from node"
+
+
+@pytest.mark.parametrize(
+ ("node", "error_type", "match"),
+ [
+ pytest.param(
+ TypedNode("array", items=TypedNode("array", items=TypedNode("string"))),
+ NestedLinkMLMultivaluedError,
+ None,
+ id="nested-array",
+ ),
+ pytest.param(
+ TypedNode("array", items=TypedNode("map", key_type=TypedNode("string"), value_type=TypedNode("string"))),
+ NestedLinkMLMultivaluedError,
+ None,
+ id="array-of-map",
+ ),
+ pytest.param(
+ TypedNode("map", key_type=None, value_type=TypedNode("string")),
+ LinkMLEmitterError,
+ "key and value",
+ id="map-missing-key",
+ ),
+ ],
+)
+def test_emit_node_fail_invalid_multivalued_and_map(
+ node: TypedNode,
+ error_type: type[Exception],
+ match: str | None,
+) -> None:
+ """emit_node rejects nested multivalued nodes and incomplete map nodes."""
+ with pytest.raises(error_type, match=match):
+ LinkMLEmitter().emit_node(node)
+
+
+def test_emit_pass_pyarrow_bounds_fixed_list_and_map() -> None:
+ """LinkMLEmitter uses document title and emits PyArrow bounds, fixed-size lists, and maps."""
+ pa_prov = Provenance("pyarrow", ())
+ pa_doc = SchemaDocument(
+ annotations={"title": "MyTitle"},
+ root=TypedNode(
+ "object",
+ properties=(
+ Field(
+ "u8_list",
+ TypedNode(
+ "array",
+ hints=NodeHints(logical_type="fixed-size-list", length=4),
+ items=TypedNode(
+ "integer",
+ hints=NodeHints(logical_type="uint8", bit_width=8),
+ provenance=pa_prov,
+ ),
+ provenance=pa_prov,
+ ),
+ ),
+ Field(
+ "i16",
+ TypedNode(
+ "integer",
+ hints=NodeHints(logical_type="int16", bit_width=16),
+ provenance=pa_prov,
+ ),
+ ),
+ Field(
+ "pa_map",
+ TypedNode(
+ "map",
+ key_type=TypedNode("string", provenance=pa_prov),
+ value_type=TypedNode("integer", nullable=False, provenance=pa_prov),
+ provenance=pa_prov,
+ ),
+ ),
+ ),
+ ),
+ )
+ out = LinkMLEmitter().emit(pa_doc)
+ assert out.name == "MyTitle"
+ assert "MyTitle" in out.classes
+ attrs = out.classes["MyTitle"].attributes
+ u8_list = attrs["u8_list"]
+ expected_len = 4
+ assert u8_list.minimum_value is None
+ assert u8_list.maximum_value is None
+ assert u8_list.minimum_cardinality == expected_len
+ assert u8_list.maximum_cardinality == expected_len
+ min_i16 = -32768
+ max_i16 = 32767
+ assert attrs["i16"].minimum_value == min_i16
+ assert attrs["i16"].maximum_value == max_i16
+ map_cls = out.classes["MyTitle_pa_map_map"]
+ assert map_cls.attributes["key"].required is True
+ assert map_cls.attributes["value"].required is True
+
+
+def test_emit_into_pass_avoids_enum_name_collision_and_unnamed_root() -> None:
+ """emit_into avoids collisions with existing enum names and handles unnamed root documents."""
+ target = SchemaDefinition(id="urn:test:ns", name="ns", enums={"Root": EnumDefinition(name="Root")})
+ LinkMLEmitter().emit_into(SchemaDocument(root=TypedNode("object")), target)
+ assert "Root_2" in target.classes
+ assert target.classes["Root_2"].description is None
diff --git a/tests/cdm_data_loaders/converters/emitters/test_pyarrow_codegen_collector.py b/tests/cdm_data_loaders/converters/emitters/test_pyarrow_codegen_collector.py
new file mode 100644
index 00000000..5d0c915f
--- /dev/null
+++ b/tests/cdm_data_loaders/converters/emitters/test_pyarrow_codegen_collector.py
@@ -0,0 +1,208 @@
+"""Unit tests for cdm_data_loaders.converters.emitters.pyarrow_codegen.collector."""
+
+from typing import Final
+
+import pyarrow as pa
+import pytest
+
+from cdm_data_loaders.converters.core.ir import Field, Provenance, SchemaDocument, TypedNode
+from cdm_data_loaders.converters.emitters.pyarrow_codegen.collector import (
+ IRComponentCollector,
+ extract_component_meta,
+ unwrap_array_element_node,
+ unwrap_map_value_node,
+ unwrap_object_node,
+)
+from cdm_data_loaders.converters.emitters.pyarrow_codegen.naming import RenderedFieldSpec
+
+PLAIN_OBJECT_NODE: Final[TypedNode] = TypedNode("object")
+ELEM_STRING_NODE: Final[TypedNode] = TypedNode("string")
+MAP_VALUE_INT_NODE: Final[TypedNode] = TypedNode("integer")
+BAD_NAME_OBJECT_NODE: Final[TypedNode] = TypedNode("object", provenance=Provenance("xsd", (), {"type_name": 123}))
+
+
+@pytest.mark.parametrize(
+ ("node", "expected"),
+ [
+ pytest.param(
+ PLAIN_OBJECT_NODE,
+ ((id(PLAIN_OBJECT_NODE), False), None, False),
+ id="plain-object-falls-back-to-node-id",
+ ),
+ pytest.param(
+ TypedNode(
+ "object",
+ provenance=Provenance(
+ "xsd",
+ (),
+ {"component_id": 7, "nillable": True, "type_name": "ItemType", "inline_struct": True},
+ ),
+ ),
+ ((7, True), "ItemType", True),
+ id="provenance-metadata-present",
+ ),
+ pytest.param(
+ BAD_NAME_OBJECT_NODE,
+ ((id(BAD_NAME_OBJECT_NODE), False), None, False),
+ id="non-string-type-name-ignored",
+ ),
+ ],
+)
+def test_extract_component_meta_variants(
+ node: TypedNode,
+ expected: tuple[tuple[int, bool], str | None, bool],
+) -> None:
+ """Verify extract_component_meta extracts component key, type name, and inline flag."""
+ assert extract_component_meta(node) == expected
+
+
+@pytest.mark.parametrize(
+ ("node", "expected"),
+ [
+ pytest.param(None, None, id="none"),
+ pytest.param(TypedNode("string"), None, id="scalar"),
+ pytest.param(PLAIN_OBJECT_NODE, PLAIN_OBJECT_NODE, id="direct-object"),
+ pytest.param(TypedNode("any", one_of=(PLAIN_OBJECT_NODE,)), PLAIN_OBJECT_NODE, id="one-of-object"),
+ pytest.param(TypedNode("any", any_of=(PLAIN_OBJECT_NODE,)), PLAIN_OBJECT_NODE, id="any-of-object"),
+ ],
+)
+def test_unwrap_object_node_variants(node: TypedNode | None, expected: TypedNode | None) -> None:
+ """Verify unwrap_object_node unwraps direct and combinator-wrapped object nodes."""
+ assert unwrap_object_node(node) is expected
+
+
+@pytest.mark.parametrize(
+ ("node", "expected"),
+ [
+ pytest.param(None, None, id="none"),
+ pytest.param(TypedNode("array", prefix_items=(ELEM_STRING_NODE,)), ELEM_STRING_NODE, id="prefix-items"),
+ pytest.param(TypedNode("array", items=(ELEM_STRING_NODE,)), ELEM_STRING_NODE, id="items-tuple"),
+ pytest.param(TypedNode("array", items=()), None, id="empty-items-tuple"),
+ pytest.param(TypedNode("array", items=ELEM_STRING_NODE), ELEM_STRING_NODE, id="single-items-node"),
+ ],
+)
+def test_unwrap_array_element_node_variants(node: TypedNode | None, expected: TypedNode | None) -> None:
+ """Verify unwrap_array_element_node extracts the element node from array variants."""
+ assert unwrap_array_element_node(node) is expected
+
+
+@pytest.mark.parametrize(
+ ("node", "expected"),
+ [
+ pytest.param(None, None, id="none"),
+ pytest.param(TypedNode("map", value_type=MAP_VALUE_INT_NODE), MAP_VALUE_INT_NODE, id="map-value-type"),
+ pytest.param(
+ TypedNode("object", pattern_properties={"^k": MAP_VALUE_INT_NODE}),
+ MAP_VALUE_INT_NODE,
+ id="pattern-properties",
+ ),
+ pytest.param(
+ TypedNode("object", additional_properties=MAP_VALUE_INT_NODE),
+ MAP_VALUE_INT_NODE,
+ id="additional-properties",
+ ),
+ pytest.param(
+ TypedNode("object", additional_properties=TypedNode("any")),
+ None,
+ id="additional-properties-any",
+ ),
+ pytest.param(
+ TypedNode("object", additional_properties=TypedNode("never")),
+ None,
+ id="additional-properties-never",
+ ),
+ ],
+)
+def test_unwrap_map_value_node_variants(node: TypedNode | None, expected: TypedNode | None) -> None:
+ """Verify unwrap_map_value_node extracts value nodes for maps and map-like objects."""
+ assert unwrap_map_value_node(node) is expected
+
+
+def test_ir_component_collector_collect_document_scalar_leaf_table() -> None:
+ """Verify collect_document returns a RenderedFieldSpec directly for scalar leaf tables."""
+ collector = IRComponentCollector()
+ scalar_doc = SchemaDocument(root=TypedNode("object", provenance=Provenance("xsd", (), {"scalar_leaf_table": True})))
+
+ scalar_spec = collector.collect_document("note", scalar_doc, pa.schema([pa.field("#text", pa.string())]))
+
+ assert isinstance(scalar_spec, RenderedFieldSpec)
+ assert scalar_spec.name == "#text"
+
+
+def test_ir_component_collector_scan_collect_and_assign_names() -> None:
+ """Verify IRComponentCollector handles truncated IDs, maps, nested lists, collisions, and inline fallback."""
+ collector = IRComponentCollector()
+
+ trunc_node = TypedNode("string", provenance=Provenance("xsd", (), {"truncated_component_id": 99}))
+ leaf_obj = TypedNode(
+ "object",
+ properties=(Field("v", TypedNode("integer")),),
+ provenance=Provenance("json-schema", (), {"component_id": 10}),
+ )
+ leaf_obj_same_key_diff_struct = TypedNode(
+ "object",
+ properties=(Field("v", TypedNode("string")),),
+ provenance=Provenance("json-schema", (), {"component_id": 10}),
+ )
+ cyclic_obj = TypedNode(
+ "object",
+ properties=(Field("name", TypedNode("string")),),
+ provenance=Provenance("xsd", (), {"component_id": 20}),
+ )
+ # self-referential properties via another node sharing component_id=20
+ cyclic_holder = TypedNode(
+ "object",
+ properties=(
+ Field("self_ref", cyclic_obj),
+ Field("trunc", trunc_node),
+ Field("nested_map", TypedNode("map", value_type=leaf_obj)),
+ Field("nested_list", TypedNode("array", items=TypedNode("array", items=leaf_obj_same_key_diff_struct))),
+ Field("empty_obj", TypedNode("object")),
+ ),
+ provenance=Provenance("xsd", (), {"component_id": 20}),
+ )
+ doc = SchemaDocument(root=cyclic_holder)
+ collector.register_document("root", doc)
+ trunc_id = 99
+ assert trunc_id in collector.el_name_types["trunc"]
+ assert "empty_obj" not in collector.el_name_types
+
+ pa_leaf_int = pa.struct([pa.field("v", pa.int64())])
+ pa_leaf_str = pa.struct([pa.field("v", pa.string())])
+ pa_schema = pa.schema(
+ [
+ pa.field("self_ref", pa.struct([pa.field("name", pa.string())])),
+ pa.field("trunc", pa.string()),
+ pa.field("nested_map", pa.map_(pa.string(), pa_leaf_int)),
+ pa.field("nested_list", pa.list_(pa.list_(pa_leaf_str))),
+ pa.field("empty_obj", pa.struct([])),
+ ]
+ )
+ root_key = collector.collect_document("root", doc, pa_schema)
+ assert root_key == (20, False)
+
+ # nillable=True on component_id=10 triggers _NILLABLE suffix
+ nillable_leaf = TypedNode(
+ "object",
+ properties=(Field("v", TypedNode("integer")),),
+ provenance=Provenance("xsd", (), {"component_id": 10, "nillable": True}),
+ )
+ collector.collect_document("nested_map", SchemaDocument(root=nillable_leaf), pa.schema([pa.field("v", pa.int64())]))
+
+ # duplicate root name without parent triggers _2_STRUCT suffix
+ dup_root = TypedNode(
+ "object",
+ properties=(Field("v", TypedNode("integer")),),
+ provenance=Provenance("xsd", (), {"component_id": 30}),
+ )
+ collector.collect_document("nested_map", SchemaDocument(root=dup_root), pa.schema([pa.field("v", pa.int64())]))
+
+ names = collector.assign_component_names()
+ assert names[(10, True)] == "NESTED_MAP_NILLABLE_STRUCT"
+ assert names[(30, False)] == "NESTED_MAP_2_STRUCT"
+ assert "__COMPONENT_10_0__" in next(
+ f.type_expr for f in collector.components[(20, False)].fields if f.name == "nested_map"
+ )
+ # differing pa_struct for same json-schema component_id=10 renders inline
+ nested_list_expr = next(f.type_expr for f in collector.components[(20, False)].fields if f.name == "nested_list")
+ assert "pa.struct([" in nested_list_expr
diff --git a/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py b/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py
index d623143f..f3d5681b 100644
--- a/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py
+++ b/tests/cdm_data_loaders/converters/facades/test_jsonschema_to_pyarrow.py
@@ -2,6 +2,7 @@
import functools
from collections.abc import Callable
+from typing import Any
import pyarrow as pa
import pytest
@@ -15,6 +16,8 @@
)
from cdm_data_loaders.converters.facades.jsonschema_to_pyarrow import (
JSONSchemaToPyArrow,
+ generate_pyarrow_schema,
+ resolve_scalar_type,
)
from cdm_data_loaders.converters.facades.pydantic_to_pyarrow import PydanticToPyArrow
from cdm_data_loaders.converters.facades.xsd_to_pyarrow import XsdToPyArrow
@@ -226,3 +229,30 @@ def test_strict_ref_helpers_fail_direct_alias_cycle(
error_type=JSONSchemaToPyArrowError,
error_by_code=ERROR_BY_CODE,
)
+
+
+@pytest.mark.parametrize(
+ ("node", "expected"),
+ [
+ pytest.param({"type": "integer"}, (pa.int64(), "integer"), id="integer"),
+ pytest.param({"type": "string", "format": "date-time"}, (pa.string(), "string (date-time)"), id="date-time"),
+ pytest.param({"enum": ["a", "b"]}, (pa.string(), "string"), id="string-enum"),
+ ],
+)
+def test_resolve_scalar_type_pass(node: dict[str, Any], expected: tuple[pa.DataType, str]) -> None:
+ """resolve_scalar_type maps scalar JSON Schema nodes to PyArrow types and labels."""
+ assert resolve_scalar_type(node) == expected
+
+
+def test_resolve_scalar_type_fail_non_scalar() -> None:
+ """resolve_scalar_type rejects non-scalar JSON Schema nodes."""
+ with pytest.raises(JSONSchemaToPyArrowError):
+ resolve_scalar_type({"type": "object", "properties": {"a": {"type": "string"}}})
+
+
+def test_generate_pyarrow_schema_pass() -> None:
+ """generate_pyarrow_schema converts a JSON Schema object into a PyArrow schema."""
+ schema = base_object_schema(properties={"x": {"type": "boolean"}})
+ assert generate_pyarrow_schema(schema) == pa.schema(
+ [pa.field("x", pa.bool_(), metadata={"jsonschema_type": "boolean"})]
+ )
diff --git a/tests/cdm_data_loaders/converters/sources/test_iceberg.py b/tests/cdm_data_loaders/converters/sources/test_iceberg.py
index 26cd1dca..c3409fba 100644
--- a/tests/cdm_data_loaders/converters/sources/test_iceberg.py
+++ b/tests/cdm_data_loaders/converters/sources/test_iceberg.py
@@ -34,6 +34,7 @@
)
from cdm_data_loaders.converters.core.errors import ConversionError
+from cdm_data_loaders.converters.core.ir import TypedNode
from cdm_data_loaders.converters.sources.iceberg import IcebergReader, iceberg_value
from tests.helpers.iceberg_catalog import make_iceberg_table as make_table
@@ -153,6 +154,20 @@ def test_read_pass_empty_schema() -> None:
"identifier",
id="empty-table-identifier",
),
+ pytest.param(
+ functools.partial(IcebergReader().read_table, make_table(("ns", "table"), Schema()), ("", "t")),
+ "nonempty strings",
+ id="empty-identifier-part",
+ ),
+ pytest.param(
+ functools.partial(
+ IcebergReader().read_table,
+ make_table(("ns", "table"), Schema()),
+ (123, "t"), # type: ignore[arg-type]
+ ),
+ "nonempty strings",
+ id="non-string-identifier-part",
+ ),
],
)
def test_read_fail_invalid_source_and_identifier(
@@ -207,3 +222,60 @@ def test_read_fail_unsupported_type_and_metadata(
"""Unmapped Iceberg types and arbitrary Python defaults are not guessed."""
with pytest.raises(error_type, match=match):
call()
+
+
+def test_read_pass_provenance_and_extensions() -> None:
+ """IcebergReader.read preserves schema provenance paths, x-iceberg metadata, and closed root additional_properties."""
+ schema = Schema(
+ NestedField(1, "id", LongType(), required=True, doc="primary id"),
+ NestedField(2, "nested", StructType(NestedField(3, "inner", StringType()))),
+ schema_id=7,
+ identifier_field_ids=[1],
+ )
+ doc = IcebergReader().read(schema)
+ expected_schema_id = 7
+ assert doc.extensions["x-iceberg"] == {"schema_id": expected_schema_id, "identifier_field_ids": (1,)}
+ assert doc.provenance is not None
+ assert doc.provenance.source == "iceberg"
+ assert doc.provenance.metadata["schema-id"] == expected_schema_id
+ assert doc.root.additional_properties == TypedNode(type="never", nullable=False)
+ id_field, nested_field = doc.root.properties
+ assert id_field.annotations == {"description": "primary id"}
+ assert id_field.provenance is not None
+ assert id_field.provenance.path == ("id",)
+ assert nested_field.node.properties[0].provenance is not None
+ assert nested_field.node.properties[0].provenance.path == ("nested", "inner")
+
+
+def test_read_node_pass_standalone_list_provenance() -> None:
+ """read_node attaches empty root provenance and element path for standalone lists."""
+ standalone_node = IcebergReader().read_node(ListType(10, StringType(), element_required=False))
+ assert standalone_node.provenance is not None
+ assert standalone_node.provenance.path == ()
+ assert standalone_node.items is not None
+ assert standalone_node.items.provenance is not None
+ assert standalone_node.items.provenance.path == ("element",)
+
+
+def test_read_field_pass_standalone_map_provenance() -> None:
+ """read_field attaches field, key, and value provenance paths for standalone maps."""
+ standalone_field = IcebergReader().read_field(NestedField(11, "col", MapType(12, StringType(), 13, IntegerType())))
+ assert standalone_field.provenance is not None
+ assert standalone_field.provenance.path == ("col",)
+ assert standalone_field.annotations == {"description": None}
+ assert standalone_field.node.key_type is not None
+ assert standalone_field.node.key_type.provenance is not None
+ assert standalone_field.node.key_type.provenance.path == ("col", "key")
+ assert standalone_field.node.value_type is not None
+ assert standalone_field.node.value_type.provenance is not None
+ assert standalone_field.node.value_type.provenance.path == ("col", "value")
+
+
+def test_read_table_pass_without_comment() -> None:
+ """read_table leaves annotations empty and records identifier and format version when no comment is set."""
+ table_no_comment = make_table(("ns", "plain"), Schema(NestedField(1, "id", LongType(), required=True)))
+ table_doc = IcebergReader().read_table(table_no_comment, ("ns", "plain"))
+ assert table_doc.annotations == {}
+ assert table_doc.provenance is not None
+ assert table_doc.provenance.path == ("ns", "plain")
+ assert "format-version" in table_doc.provenance.metadata
diff --git a/tests/cdm_data_loaders/converters/sources/test_pyarrow_reader.py b/tests/cdm_data_loaders/converters/sources/test_pyarrow_reader.py
new file mode 100644
index 00000000..bb72ee4a
--- /dev/null
+++ b/tests/cdm_data_loaders/converters/sources/test_pyarrow_reader.py
@@ -0,0 +1,291 @@
+"""Unit tests for cdm_data_loaders.converters.sources.pyarrow.PyArrowReader."""
+
+from typing import Any, Final
+
+import pyarrow as pa
+import pytest
+
+from cdm_data_loaders.converters.core.ir import NodeHints, Provenance, TypedNode
+from cdm_data_loaders.converters.exceptions.sources.pyarrow import PyArrowReaderError
+from cdm_data_loaders.converters.sources.pyarrow import PyArrowReader, _format_path
+
+
+class _DummyExtType(pa.ExtensionType):
+ def __init__(self) -> None:
+ super().__init__(pa.int64(), "dummy.ext")
+
+ def __arrow_ext_serialize__(self) -> bytes:
+ return b""
+
+ @classmethod
+ def __arrow_ext_deserialize__(cls, _storage_type: pa.DataType, _serialized: bytes) -> "_DummyExtType":
+ return cls()
+
+
+LIST_ELEM_FIELD: Final[pa.Field] = pa.field(
+ "item", pa.int64(), nullable=False, metadata={"comment": "elem desc", "k": "v"}
+)
+
+
+@pytest.mark.parametrize(
+ ("path", "expected"),
+ [
+ pytest.param((), "", id="empty-root"),
+ pytest.param(("a", 0, "b"), "a.0.b", id="nested-segments"),
+ ],
+)
+def test_format_path_segments(path: tuple[str | int, ...], expected: str) -> None:
+ """Verify _format_path formats empty and nested path tuples."""
+ assert _format_path(path) == expected
+
+
+@pytest.mark.parametrize(
+ ("bad_schema", "pattern"),
+ [
+ pytest.param("not_a_schema", r"Expected a pyarrow\.Schema", id="non-schema"),
+ pytest.param(pa.schema([]), "Empty PyArrow schema with 0 fields", id="empty-schema"),
+ pytest.param(
+ pa.schema([pa.field("dup", pa.int64()), pa.field("dup", pa.string())]),
+ "Duplicate field name 'dup' in PyArrow schema",
+ id="duplicate-field",
+ ),
+ ],
+)
+def test_pyarrow_reader_read_invalid_input_raises(bad_schema: object, pattern: str) -> None:
+ """Verify PyArrowReader.read rejects invalid, empty, or duplicate-field schemas."""
+ with pytest.raises(PyArrowReaderError, match=pattern):
+ PyArrowReader().read(bad_schema) # type: ignore[arg-type]
+
+
+def test_pyarrow_reader_read_field_invalid_type_raises() -> None:
+ """Verify PyArrowReader.read_field rejects non-Field inputs."""
+ with pytest.raises(PyArrowReaderError, match=r"Expected a pyarrow\.Field"):
+ PyArrowReader().read_field("not_a_field") # type: ignore[arg-type]
+
+
+@pytest.mark.parametrize(
+ ("bad_type", "path", "pattern"),
+ [
+ pytest.param("not_a_type", (), r"Expected a pyarrow\.DataType", id="non-datatype"),
+ pytest.param(pa.struct([]), ("nested",), "Empty PyArrow struct with 0 fields", id="empty-struct"),
+ pytest.param(
+ pa.struct([pa.field("x", pa.int64()), pa.field("x", pa.string())]),
+ ("nested",),
+ "Duplicate field name 'x' in PyArrow struct at 'nested'",
+ id="duplicate-struct-field",
+ ),
+ pytest.param(pa.null(), ("col",), "Standalone PyArrow null type", id="standalone-null"),
+ pytest.param(_DummyExtType(), ("col",), "Unsupported PyArrow extension type", id="extension-type"),
+ pytest.param(pa.duration("s"), ("col",), "duration/interval", id="duration"),
+ pytest.param(pa.month_day_nano_interval(), ("col",), "duration/interval", id="interval"),
+ pytest.param(pa.union([pa.field("a", pa.int32())], mode="sparse"), ("col",), "union type", id="union"),
+ pytest.param(pa.run_end_encoded(pa.int32(), pa.string()), ("col",), "run_end_encoded", id="run-end-encoded"),
+ pytest.param(pa.decimal128(5, -1), ("col",), "Invalid PyArrow decimal", id="decimal-negative-scale"),
+ pytest.param(pa.decimal128(5, 6), ("col",), "Invalid PyArrow decimal", id="decimal-scale-exceeds-precision"),
+ ],
+)
+def test_pyarrow_reader_read_node_invalid_input_raises(
+ bad_type: object,
+ path: tuple[str | int, ...],
+ pattern: str,
+) -> None:
+ """Verify PyArrowReader.read_node rejects unsupported and malformed PyArrow data types."""
+ with pytest.raises(PyArrowReaderError, match=pattern):
+ PyArrowReader().read_node(bad_type, path=path) # type: ignore[arg-type]
+
+
+def test_pyarrow_reader_read_preserves_metadata_and_requiredness() -> None:
+ """Verify PyArrowReader.read preserves schema metadata, annotations, provenance, and required fields."""
+ schema = pa.schema(
+ [
+ pa.field("req_id", pa.int64(), nullable=False, metadata={"comment": "ID", "pydantic_type": "int"}),
+ pa.field("opt_name", pa.string(), nullable=True, metadata={"custom": "v"}),
+ ],
+ metadata={"title": "MetaTitle", "$id": "https://example.org/meta", "comment": "Doc desc", "extra": "keep"},
+ )
+ doc = PyArrowReader().read(schema)
+
+ assert doc.name == "MetaTitle"
+ assert doc.identifier == "https://example.org/meta"
+ assert doc.annotations["description"] == "Doc desc"
+ assert doc.root.annotations["description"] == "Doc desc"
+ assert doc.extensions["x-pyarrow"]["metadata"] == {"extra": "keep"}
+ assert doc.root.required_names == ("req_id",)
+ assert doc.root.additional_properties == TypedNode(type="never", nullable=False)
+ assert doc.provenance == Provenance("pyarrow", ())
+ assert doc.root.provenance == Provenance("pyarrow", ())
+
+
+def test_pyarrow_reader_read_overrides_name_and_id() -> None:
+ """Verify PyArrowReader.read uses explicit schema_name and schema_id overrides over metadata."""
+ schema = pa.schema(
+ [pa.field("a", pa.int32())],
+ metadata={"title": "MetaTitle", "$id": "https://example.org/meta"},
+ )
+ doc = PyArrowReader().read(schema, schema_name="OverrideName", schema_id="https://example.org/override")
+
+ assert doc.name == "OverrideName"
+ assert doc.identifier == "https://example.org/override"
+
+
+def test_pyarrow_reader_read_excludes_metadata_when_disabled() -> None:
+ """Verify PyArrowReader omits x-pyarrow metadata when include_pyarrow_metadata is False."""
+ doc = PyArrowReader(include_pyarrow_metadata=False).read(pa.schema([pa.field("a", pa.int32())]))
+
+ assert doc.name is None
+ assert doc.identifier is None
+ assert not doc.annotations
+ assert not doc.extensions
+
+
+def test_pyarrow_reader_read_field_metadata_and_provenance() -> None:
+ """Verify PyArrowReader.read_field preserves requiredness, description, provenance, and extensions."""
+ field = PyArrowReader().read_field(
+ pa.field("col", pa.int32(), nullable=False, metadata={"comment": "c", "xsd_type": "xs:int"})
+ )
+
+ assert field.required is True
+ assert field.annotations["description"] == "c"
+ assert field.provenance == Provenance("pyarrow", ("col",), {"xsd_type": "xs:int"})
+ assert field.extensions["x-pyarrow"]["metadata"] == {"xsd_type": "xs:int"}
+
+
+@pytest.mark.parametrize(
+ ("list_type", "expected_arrow"),
+ [
+ pytest.param(pa.list_(LIST_ELEM_FIELD), "list", id="list"),
+ pytest.param(pa.large_list(LIST_ELEM_FIELD), "large_list", id="large-list"),
+ pytest.param(pa.list_view(LIST_ELEM_FIELD), "list_view", id="list-view"),
+ pytest.param(pa.large_list_view(LIST_ELEM_FIELD), "large_list_view", id="large-list-view"),
+ pytest.param(pa.list_(LIST_ELEM_FIELD, 3), "fixed_size_list", id="fixed-size-list"),
+ ],
+)
+@pytest.mark.parametrize(
+ "include_metadata",
+ [
+ pytest.param(True, id="with-metadata"),
+ pytest.param(False, id="without-metadata"),
+ ],
+)
+def test_pyarrow_reader_read_node_list_variants(
+ list_type: pa.DataType,
+ expected_arrow: str,
+ include_metadata: bool,
+) -> None:
+ """Verify PyArrowReader.read_node handles all PyArrow list variants with and without metadata."""
+ node = PyArrowReader(include_pyarrow_metadata=include_metadata).read_node(list_type, nullable=True, path=("arr",))
+
+ assert node.type == "array"
+ assert node.nullable is True
+ assert isinstance(node.items, TypedNode)
+ assert node.items.annotations["description"] == "elem desc"
+ if include_metadata:
+ assert node.extensions["x-pyarrow"]["arrow_type"] == expected_arrow
+ assert node.items.extensions["x-pyarrow"]["metadata"] == {"k": "v"}
+ else:
+ assert not node.items.extensions
+
+
+@pytest.mark.parametrize(
+ ("include_metadata", "ordered"),
+ [
+ pytest.param(True, True, id="ordered-with-metadata"),
+ pytest.param(False, False, id="unordered-without-metadata"),
+ ],
+)
+def test_pyarrow_reader_read_node_dictionary(include_metadata: bool, ordered: bool) -> None:
+ """Verify PyArrowReader.read_node lowers dictionary types and preserves ordering metadata when enabled."""
+ node = PyArrowReader(include_pyarrow_metadata=include_metadata).read_node(
+ pa.dictionary(pa.int8(), pa.string(), ordered=ordered),
+ nullable=True,
+ )
+
+ assert node.type == "string"
+ assert node.nullable is True
+ if include_metadata:
+ assert node.extensions["x-pyarrow"]["arrow_type"] == "dictionary"
+ assert node.extensions["x-pyarrow"]["ordered"] is True
+ else:
+ assert not node.extensions
+
+
+def test_pyarrow_reader_read_node_map() -> None:
+ """Verify PyArrowReader.read_node lowers PyArrow map types with non-null keys and keys_sorted metadata."""
+ node = PyArrowReader().read_node(pa.map_(pa.string(), pa.int32(), keys_sorted=True))
+
+ assert node.type == "map"
+ assert node.key_type is not None
+ assert node.key_type.nullable is False
+ assert node.extensions["x-pyarrow"]["keys_sorted"] is True
+
+
+def test_pyarrow_reader_read_node_struct() -> None:
+ """Verify PyArrowReader.read_node lowers PyArrow struct types and tracks required fields."""
+ node = PyArrowReader().read_node(pa.struct([pa.field("a", pa.bool_(), nullable=False)]))
+
+ assert node.type == "object"
+ assert node.required_names == ("a",)
+
+
+@pytest.mark.parametrize(
+ ("arrow_type", "expected_hints", "expected_arrow_meta", "absent_meta_keys"),
+ [
+ pytest.param(
+ pa.decimal256(40, 4),
+ NodeHints(logical_type="decimal", precision=40, scale=4, bit_width=256),
+ {},
+ (),
+ id="decimal256",
+ ),
+ pytest.param(
+ pa.binary(8),
+ NodeHints(logical_type="fixed", length=8),
+ {},
+ (),
+ id="fixed-size-binary",
+ ),
+ pytest.param(
+ pa.time32("s"),
+ None,
+ {"arrow_type": "time32"},
+ (),
+ id="time32",
+ ),
+ pytest.param(
+ pa.time64("ns"),
+ None,
+ {"arrow_type": "time64"},
+ (),
+ id="time64",
+ ),
+ pytest.param(
+ pa.timestamp("us"),
+ NodeHints(logical_type="timestamp-without-tz", timezone=False),
+ {},
+ ("tz",),
+ id="timestamp-without-tz",
+ ),
+ pytest.param(
+ pa.timestamp("us", tz="UTC"),
+ NodeHints(logical_type="timestamp-with-tz", timezone=True),
+ {"tz": "UTC"},
+ (),
+ id="timestamp-with-tz",
+ ),
+ ],
+)
+def test_pyarrow_reader_read_node_scalars(
+ arrow_type: pa.DataType,
+ expected_hints: NodeHints | None,
+ expected_arrow_meta: dict[str, Any],
+ absent_meta_keys: tuple[str, ...],
+) -> None:
+ """Verify PyArrowReader.read_node maps scalar PyArrow types to hints and x-pyarrow metadata."""
+ node = PyArrowReader().read_node(arrow_type)
+
+ if expected_hints is not None:
+ assert node.hints == expected_hints
+ for key, val in expected_arrow_meta.items():
+ assert node.extensions["x-pyarrow"][key] == val
+ for key in absent_meta_keys:
+ assert key not in node.extensions["x-pyarrow"]
diff --git a/tests/cdm_data_loaders/converters/sources/test_pydantic_annotations.py b/tests/cdm_data_loaders/converters/sources/test_pydantic_annotations.py
new file mode 100644
index 00000000..ef0395a2
--- /dev/null
+++ b/tests/cdm_data_loaders/converters/sources/test_pydantic_annotations.py
@@ -0,0 +1,279 @@
+"""Unit tests for Pydantic annotation unwrapping and classification in cdm_data_loaders.converters.sources.pydantic.annotations."""
+
+import re
+from collections.abc import Callable
+from decimal import Decimal
+from enum import StrEnum
+from typing import Annotated, Any, Final, Literal, NewType, TypeVar, Unpack
+
+import pytest
+from pydantic import Base64Bytes, BaseModel, ConfigDict, Field, NaiveDatetime, RootModel, computed_field
+
+from cdm_data_loaders.converters.core.errors import ConversionError
+from cdm_data_loaders.converters.sources.pydantic.annotations import (
+ ParsedAnnotation,
+ allows_null,
+ classify_parsed_kind,
+ extract_model_field_specs,
+ is_valid_string_map_key,
+ is_variadic_unpack,
+ substitute_typevars,
+ unwrap_annotation,
+)
+from cdm_data_loaders.converters.sources.pydantic.model_synthesis import SchemaBuildContext
+
+type MyBox[T] = list[T]
+type SimpleAlias = Annotated[int, Field(description="from alias")]
+UserId = NewType("UserId", str)
+SAMPLE_TYPEVAR: Final = TypeVar("SAMPLE_TYPEVAR")
+
+
+class _Color(StrEnum):
+ RED = "red"
+
+
+class _DocModel(BaseModel):
+ """Docstring from model."""
+
+ x: int
+
+
+class _RootStr(RootModel[str]):
+ pass
+
+
+class _ExtraAllowModel(BaseModel):
+ model_config = ConfigDict(extra="allow")
+ x: int
+
+
+class _RichModel(BaseModel):
+ kept: int = Field(alias="keptAlias", serialization_alias="keptSer", description="kept desc")
+ hidden: str = Field(default="x", exclude=True)
+ opt: str | None = None
+
+ @computed_field(alias="compAlias", description="comp desc") # type: ignore[prop-decorator]
+ @property
+ def total(self) -> int:
+ return self.kept
+
+
+class _AliasFallbackModel(BaseModel):
+ a_field: int = Field(alias="aAlias")
+ default_val: int = 5
+ default_none: int = None # type: ignore[assignment]
+
+ @computed_field(alias="cAlias") # type: ignore[prop-decorator]
+ @property
+ def comp_opt(self) -> str | None:
+ return None
+
+
+def _ctx(*, by_alias: bool = False, module_ns: dict[str, Any] | None = None) -> SchemaBuildContext:
+ return SchemaBuildContext(
+ model_counts={},
+ max_recursion_depth=3,
+ by_alias=by_alias,
+ error_type=ConversionError,
+ error_by_code={},
+ module_ns=module_ns or {},
+ )
+
+
+@pytest.mark.parametrize(
+ ("annotation", "mapping", "expected"),
+ [
+ pytest.param(SAMPLE_TYPEVAR, {SAMPLE_TYPEVAR: int}, int, id="direct-typevar"),
+ pytest.param(Annotated[SAMPLE_TYPEVAR, "m"], {SAMPLE_TYPEVAR: str}, Annotated[str, "m"], id="annotated"),
+ pytest.param(SAMPLE_TYPEVAR | None, {SAMPLE_TYPEVAR: int}, int | None, id="union"),
+ pytest.param(list[SAMPLE_TYPEVAR], {SAMPLE_TYPEVAR: float}, list[float], id="generic-list"),
+ ],
+)
+def test_substitute_typevars_replaces_parameters(
+ annotation: object,
+ mapping: dict[TypeVar, Any],
+ expected: object,
+) -> None:
+ """Verify substitute_typevars replaces TypeVars across direct, annotated, union, and generic forms."""
+ assert substitute_typevars(annotation, mapping) == expected
+
+
+@pytest.mark.parametrize(
+ ("annotation", "expected_attrs"),
+ [
+ pytest.param(MyBox[int], {"annotation": list[int]}, id="generic-type-alias"),
+ pytest.param(SimpleAlias, {"annotation": int, "description": "from alias"}, id="annotated-type-alias"),
+ pytest.param("TargetStr", {"annotation": str}, id="forward-ref"),
+ pytest.param(UserId, {"annotation": str}, id="new-type"),
+ pytest.param(_RootStr, {"annotation": str}, id="root-model"),
+ pytest.param(
+ Annotated[Decimal, Field(max_digits=10, decimal_places=3)],
+ {"max_digits": 10, "decimal_places": 3},
+ id="decimal-constraints",
+ ),
+ pytest.param(NaiveDatetime, {"is_naive_datetime": True}, id="naive-datetime"),
+ pytest.param(Base64Bytes, {"is_base64_bytes": True}, id="base64-bytes"),
+ pytest.param(_DocModel | None, {"description": "Docstring from model."}, id="optional-model-docstring"),
+ ],
+)
+def test_unwrap_annotation_variants(annotation: object, expected_attrs: dict[str, Any]) -> None:
+ """Verify unwrap_annotation resolves aliases, forward refs, NewType, RootModel, and metadata."""
+ parsed, _ = unwrap_annotation(annotation, _ctx(module_ns={"TargetStr": str}))
+ for attr, expected in expected_attrs.items():
+ assert getattr(parsed, attr) == expected
+
+
+def test_unwrap_annotation_missing_forward_ref_raises() -> None:
+ """Verify unwrap_annotation raises ConversionError for unresolved forward references."""
+ with pytest.raises(ConversionError):
+ unwrap_annotation("MissingRef", _ctx(module_ns={"TargetStr": str}))
+
+
+@pytest.mark.parametrize(
+ ("annotation", "expected"),
+ [
+ pytest.param(None, True, id="none"),
+ pytest.param(Literal.__getitem__((None, "a")), True, id="literal-with-none"),
+ pytest.param(int | None, True, id="optional-int"),
+ pytest.param(str, False, id="non-null-str"),
+ ],
+)
+def test_allows_null_variants(annotation: object, expected: bool) -> None:
+ """Verify allows_null detects nullable annotations and rejects non-nullable ones."""
+ assert allows_null(annotation, _ctx()) is expected
+
+
+@pytest.mark.parametrize(
+ ("annotation", "expected_kind"),
+ [
+ pytest.param(None, "null", id="null"),
+ pytest.param(list[int], "sequence", id="list"),
+ pytest.param(tuple[int, ...], "sequence", id="variadic-tuple"),
+ pytest.param(tuple[int, str], "tuple", id="fixed-tuple"),
+ pytest.param(dict[str, int], "map", id="dict-map"),
+ pytest.param(_DocModel, "model", id="model"),
+ pytest.param(int, "scalar", id="scalar"),
+ ],
+)
+def test_classify_parsed_kind_valid(annotation: object, expected_kind: str) -> None:
+ """Verify classify_parsed_kind classifies supported annotations into IR kinds."""
+ assert classify_parsed_kind(ParsedAnnotation(annotation), _ctx()) == expected_kind
+
+
+@pytest.mark.parametrize(
+ "bad_annotation",
+ [
+ pytest.param(Any, id="any"),
+ pytest.param(object, id="object"),
+ pytest.param(TypeVar("U"), id="typevar"),
+ pytest.param(Callable[[], None], id="callable"),
+ pytest.param(re.Pattern, id="regex-pattern"),
+ pytest.param(list, id="untyped-list"),
+ pytest.param(tuple, id="untyped-tuple"),
+ pytest.param(dict, id="untyped-dict"),
+ pytest.param(BaseModel, id="bare-base-model"),
+ pytest.param(RootModel, id="bare-root-model"),
+ ],
+)
+def test_classify_parsed_kind_unsupported_raises(bad_annotation: object) -> None:
+ """Verify classify_parsed_kind raises ConversionError for unsupported annotations."""
+ with pytest.raises(ConversionError):
+ classify_parsed_kind(ParsedAnnotation(bad_annotation), _ctx())
+
+
+@pytest.mark.parametrize(
+ ("key_type", "expected"),
+ [
+ pytest.param(str, True, id="str"),
+ pytest.param(Literal["a", _Color.RED], True, id="literal-str-and-strenum"),
+ pytest.param(_Color | Literal["b"] | None, True, id="strenum-literal-optional-union"),
+ pytest.param(int, False, id="int"),
+ pytest.param(None, False, id="none"),
+ pytest.param(Annotated[None, "a"] | Annotated[None, "b"], False, id="annotated-none-union"),
+ pytest.param(Literal[1, "a"], False, id="mixed-literal"),
+ ],
+)
+def test_is_valid_string_map_key_variants(key_type: object, expected: bool) -> None:
+ """Verify is_valid_string_map_key accepts string-like keys and rejects non-string keys."""
+ assert is_valid_string_map_key(key_type, _ctx()) is expected
+
+
+@pytest.mark.parametrize(
+ "bad_key",
+ [
+ pytest.param(Any, id="any"),
+ pytest.param(object, id="object"),
+ pytest.param(TypeVar("K"), id="typevar"),
+ ],
+)
+def test_is_valid_string_map_key_unsupported_raises(bad_key: object) -> None:
+ """Verify is_valid_string_map_key raises ConversionError for unsupported key types."""
+ with pytest.raises(ConversionError):
+ is_valid_string_map_key(bad_key, _ctx())
+
+
+@pytest.mark.parametrize(
+ ("annotation", "expected"),
+ [
+ pytest.param(Ellipsis, True, id="ellipsis"),
+ pytest.param(Unpack[tuple[int, ...]], True, id="unpack-variadic-tuple"),
+ pytest.param(Unpack[tuple], True, id="unpack-untyped-tuple"),
+ pytest.param(Unpack[tuple[int, str]], False, id="unpack-fixed-tuple"),
+ pytest.param(int, False, id="scalar-int"),
+ ],
+)
+def test_is_variadic_unpack_variants(annotation: object, expected: bool) -> None:
+ """Verify is_variadic_unpack distinguishes variadic tuple unpacks from fixed types."""
+ assert is_variadic_unpack(annotation) is expected
+
+
+def test_extract_model_field_specs_extra_allow_raises() -> None:
+ """Verify extract_model_field_specs rejects models with extra='allow'."""
+ with pytest.raises(ConversionError):
+ extract_model_field_specs(_ExtraAllowModel, _ctx())
+
+
+def test_extract_model_field_specs_excluded_aliases_and_computed() -> None:
+ """Verify extract_model_field_specs omits excluded fields and resolves serialization and computed aliases."""
+ specs = extract_model_field_specs(_RichModel, _ctx(by_alias=True))
+ by_name = {s.output_name: s for s in specs}
+
+ assert "hidden" not in by_name
+ assert by_name["keptSer"].required_non_null is True
+ assert by_name["keptSer"].description == "kept desc"
+ assert by_name["opt"].required_non_null is False
+ assert by_name["compAlias"].annotation is int
+ assert by_name["compAlias"].description == "comp desc"
+
+
+@pytest.mark.parametrize(
+ ("by_alias", "expected_names", "expected_required"),
+ [
+ pytest.param(
+ False,
+ {"a_field", "default_val", "default_none", "comp_opt"},
+ {"a_field": True, "default_val": True, "default_none": False, "comp_opt": False},
+ id="python-names",
+ ),
+ pytest.param(
+ True,
+ {"aAlias", "default_val", "default_none", "cAlias"},
+ {"aAlias": True, "default_val": True, "default_none": False, "cAlias": False},
+ id="alias-names",
+ ),
+ ],
+)
+def test_extract_model_field_specs_alias_mode_and_defaults(
+ by_alias: bool,
+ expected_names: set[str],
+ expected_required: dict[str, bool],
+) -> None:
+ """Verify extract_model_field_specs honors by_alias and marks default=None fields nullable."""
+ specs = extract_model_field_specs(_AliasFallbackModel, _ctx(by_alias=by_alias))
+ by_name = {s.output_name: s for s in specs}
+
+ assert set(by_name) == expected_names
+ for name, req in expected_required.items():
+ assert by_name[name].required_non_null is req
+ first_spec = next(iter(by_name.values()))
+ assert first_spec.context.model_counts[_AliasFallbackModel] == 1
diff --git a/tests/cdm_data_loaders/converters/sources/test_pydantic_model_synthesis.py b/tests/cdm_data_loaders/converters/sources/test_pydantic_model_synthesis.py
new file mode 100644
index 00000000..db30429e
--- /dev/null
+++ b/tests/cdm_data_loaders/converters/sources/test_pydantic_model_synthesis.py
@@ -0,0 +1,374 @@
+"""Unit tests for cdm_data_loaders.converters.sources.pydantic.model_synthesis."""
+
+from dataclasses import dataclass, field
+from enum import Enum, IntEnum, StrEnum
+from typing import Any, ForwardRef, NotRequired, Required, TypedDict
+
+import pytest
+from frozendict import frozendict
+from pydantic import BaseModel, PydanticSchemaGenerationError, RootModel, create_model
+
+from cdm_data_loaders.converters.core.errors import ConversionError
+from cdm_data_loaders.converters.sources.pydantic.model_synthesis import (
+ SchemaBuildContext,
+ clean_docstring,
+ coerce_structured_type_to_model,
+ dataclass_to_model,
+ ensure_concrete_model,
+ ensure_model_complete,
+ extract_class_docstring,
+ safe_type_hints,
+ typeddict_to_model,
+ validate_model_class,
+)
+
+
+class CustomConvError(ConversionError):
+ """Custom conversion error for context tests."""
+
+
+class SampleModel(BaseModel):
+ """Sample model doc."""
+
+ x: int
+
+
+class SampleEnum(Enum):
+ """Sample enum doc."""
+
+ A = "a"
+
+
+class SampleIntEnum(IntEnum):
+ """Sample int enum doc."""
+
+ ONE = 1
+
+
+class SampleStrEnum(StrEnum):
+ """Sample str enum doc."""
+
+ FOO = "foo"
+
+
+class _NoDocChildModel(SampleModel):
+ y: str
+
+
+class SampleTypedDict(TypedDict, total=False):
+ """TypedDict doc."""
+
+ req_field: Required[int]
+ opt_field: NotRequired[str]
+ plain_opt: float
+
+
+class _TotalTypedDict(TypedDict):
+ a: int
+
+
+@dataclass
+class SampleDataclass:
+ """Dataclass doc.
+
+ More details.
+ """
+
+ req: int
+ with_default: str = "abc"
+ with_factory: list[int] = field(default_factory=list)
+
+
+@dataclass
+class _AutoDocDataclass:
+ x: int
+
+
+class _UnresolvedHints:
+ pass
+
+
+_UnresolvedHints.__annotations__ = {"x": "MissingHintType"}
+
+
+def _error_ctx() -> SchemaBuildContext:
+ return SchemaBuildContext(
+ error_by_code=frozendict({"E1": CustomConvError, "E4": CustomConvError, "E9": CustomConvError})
+ )
+
+
+@pytest.mark.parametrize(
+ ("code", "message", "cause", "expected_exc"),
+ [
+ pytest.param("E4", "bad generic", ValueError("boom"), CustomConvError, id="mapped-code-with-cause"),
+ pytest.param("E99", "other", None, ConversionError, id="unmapped-code-fallback"),
+ ],
+)
+def test_schema_build_context_raise_err(
+ code: str,
+ message: str,
+ cause: Exception | None,
+ expected_exc: type[ConversionError],
+) -> None:
+ """Verify SchemaBuildContext.raise_err selects mapped or fallback errors and preserves cause."""
+ ctx = SchemaBuildContext(
+ max_recursion_depth=5,
+ by_alias=True,
+ error_type=ConversionError,
+ error_by_code=frozendict({"E4": CustomConvError}),
+ )
+
+ with pytest.raises(expected_exc, match=message) as exc_info:
+ ctx.raise_err(code, message, cause=cause)
+ assert exc_info.value.__cause__ is cause
+
+
+def test_schema_build_context_with_alias_and_with_model() -> None:
+ """Verify with_alias and with_model update active aliases, model counts, and module namespace immutably."""
+ ctx = SchemaBuildContext(
+ max_recursion_depth=5,
+ by_alias=True,
+ error_type=ConversionError,
+ error_by_code=frozendict({"E4": CustomConvError}),
+ )
+
+ alias_ctx = ctx.with_alias("AliasObj")
+ assert alias_ctx.active_aliases == frozenset({"AliasObj"})
+ assert alias_ctx.max_recursion_depth == ctx.max_recursion_depth
+ assert alias_ctx.by_alias is True
+ assert alias_ctx.error_by_code == ctx.error_by_code
+
+ model_ctx = alias_ctx.with_model(SampleModel)
+ assert model_ctx.model_counts[SampleModel] == 1
+ assert model_ctx.active_aliases == frozenset()
+ assert model_ctx.max_recursion_depth == ctx.max_recursion_depth
+ assert model_ctx.by_alias is True
+ assert model_ctx.error_by_code == ctx.error_by_code
+ assert "SampleModel" in model_ctx.module_ns
+
+ model_ctx2 = model_ctx.with_model(SampleModel)
+ expected_count = 2
+ assert model_ctx2.model_counts[SampleModel] == expected_count
+
+
+def test_schema_build_context_with_model_missing_module_preserves_ns() -> None:
+ """Verify with_model keeps the existing module_ns when the model module is not in sys.modules."""
+
+ class MissingModModel(BaseModel):
+ a: int
+
+ MissingModModel.__module__ = "nonexistent_mod_xyz_123"
+ custom_ns = frozendict({"sentinel": 42})
+ ctx_with_ns = SchemaBuildContext(module_ns=custom_ns)
+
+ assert ctx_with_ns.with_model(MissingModModel).module_ns == custom_ns
+
+
+def test_validate_model_class_valid() -> None:
+ """Verify validate_model_class returns a concrete BaseModel subclass unchanged."""
+ assert validate_model_class(SampleModel, _error_ctx()) is SampleModel
+
+
+@pytest.mark.parametrize(
+ "invalid",
+ [
+ pytest.param(123, id="non-type"),
+ pytest.param(dict, id="non-model-class"),
+ pytest.param(BaseModel, id="bare-base-model"),
+ pytest.param(RootModel, id="bare-root-model"),
+ ],
+)
+def test_validate_model_class_invalid_raises(invalid: object) -> None:
+ """Verify validate_model_class rejects non-model types and abstract Pydantic base classes."""
+ with pytest.raises(CustomConvError):
+ validate_model_class(invalid, _error_ctx())
+
+
+def test_ensure_model_complete_rebuilds_incomplete_model() -> None:
+ """Verify ensure_model_complete rebuilds an incomplete model with resolvable forward references."""
+
+ class IncompleteModel(BaseModel):
+ ref: "SampleModel"
+
+ IncompleteModel.__pydantic_complete__ = False
+ ensure_model_complete(IncompleteModel, _error_ctx())
+
+ assert IncompleteModel.__pydantic_complete__ is True
+
+
+@pytest.mark.parametrize(
+ ("ctx", "expected_exc"),
+ [
+ pytest.param(_error_ctx(), CustomConvError, id="with-context"),
+ pytest.param(None, ConversionError, id="without-context"),
+ ],
+)
+def test_ensure_model_complete_unresolved_forward_ref_raises(
+ ctx: SchemaBuildContext | None,
+ expected_exc: type[ConversionError],
+) -> None:
+ """Verify ensure_model_complete raises on unresolved forward references."""
+ broken_ref_model = create_model("BrokenRefModel", bad=(ForwardRef("DoesNotExistXYZ"), ...))
+
+ with pytest.raises(expected_exc, match="Unresolved forward reference"):
+ ensure_model_complete(broken_ref_model, ctx)
+
+
+def test_ensure_model_complete_ignores_schema_generation_error(monkeypatch: pytest.MonkeyPatch) -> None:
+ """Verify ensure_model_complete ignores PydanticSchemaGenerationError during model_rebuild."""
+
+ class SchemaGenErrModel(BaseModel):
+ x: int
+
+ def _raise_schema_gen(_cls: type[BaseModel], **_kwargs: object) -> bool:
+ msg = "skip"
+ raise PydanticSchemaGenerationError(msg)
+
+ SchemaGenErrModel.__pydantic_complete__ = False
+ monkeypatch.setattr(SchemaGenErrModel, "model_rebuild", classmethod(_raise_schema_gen))
+
+ ensure_model_complete(SchemaGenErrModel, _error_ctx())
+
+
+@pytest.mark.parametrize(
+ ("ctx", "expected_exc"),
+ [
+ pytest.param(_error_ctx(), CustomConvError, id="with-context"),
+ pytest.param(None, ConversionError, id="without-context"),
+ ],
+)
+def test_ensure_model_complete_still_incomplete_raises(
+ monkeypatch: pytest.MonkeyPatch,
+ ctx: SchemaBuildContext | None,
+ expected_exc: type[ConversionError],
+) -> None:
+ """Verify ensure_model_complete raises when model_rebuild returns False."""
+
+ class StillIncompleteModel(BaseModel):
+ x: int
+
+ def _return_false(_cls: type[BaseModel], **_kwargs: object) -> bool:
+ return False
+
+ StillIncompleteModel.__pydantic_complete__ = False
+ monkeypatch.setattr(StillIncompleteModel, "model_rebuild", classmethod(_return_false))
+
+ with pytest.raises(expected_exc, match="remains incomplete"):
+ ensure_model_complete(StillIncompleteModel, ctx)
+
+
+def test_ensure_concrete_model_unparameterized_generic_raises() -> None:
+ """Verify ensure_concrete_model rejects unparameterized generic models."""
+
+ class GenericBox[T](BaseModel):
+ val: T
+
+ with pytest.raises(CustomConvError, match="Unparameterized generic model"):
+ ensure_concrete_model(GenericBox, _error_ctx())
+
+
+@pytest.mark.parametrize(
+ ("cls", "expected"),
+ [
+ pytest.param(SampleModel, "Sample model doc.", id="model-with-doc"),
+ pytest.param(_NoDocChildModel, None, id="subclass-without-own-doc"),
+ pytest.param(_AutoDocDataclass, None, id="auto-generated-dataclass-doc"),
+ pytest.param(SampleDataclass, "Dataclass doc.\n\nMore details.", id="multiline-dataclass-doc"),
+ ],
+)
+def test_clean_docstring_variants(cls: type[Any], expected: str | None) -> None:
+ """Verify clean_docstring strips indentation and ignores inherited or auto-generated docstrings."""
+ assert clean_docstring(cls) == expected
+
+
+@pytest.mark.parametrize(
+ ("target", "expected"),
+ [
+ pytest.param("not a type", None, id="non-type"),
+ pytest.param(dict, None, id="builtin-dict"),
+ pytest.param(BaseModel, None, id="base-model"),
+ pytest.param(RootModel, None, id="root-model"),
+ pytest.param(Enum, None, id="enum-base"),
+ pytest.param(IntEnum, None, id="int-enum-base"),
+ pytest.param(StrEnum, None, id="str-enum-base"),
+ pytest.param(SampleModel, "Sample model doc.", id="sample-model"),
+ pytest.param(SampleEnum, "Sample enum doc.", id="sample-enum"),
+ pytest.param(SampleIntEnum, "Sample int enum doc.", id="sample-int-enum"),
+ pytest.param(SampleStrEnum, "Sample str enum doc.", id="sample-str-enum"),
+ ],
+)
+def test_extract_class_docstring_variants(target: object, expected: str | None) -> None:
+ """Verify extract_class_docstring returns user docstrings and skips non-types and base classes."""
+ assert extract_class_docstring(target) == expected
+
+
+@pytest.mark.parametrize(
+ ("cls", "expected"),
+ [
+ pytest.param(_UnresolvedHints, {"x": "MissingHintType"}, id="unresolved-annotations-fallback"),
+ pytest.param(_TotalTypedDict, {"a": int}, id="resolved-typeddict"),
+ ],
+)
+def test_safe_type_hints_variants(cls: type[Any], expected: dict[str, Any]) -> None:
+ """Verify safe_type_hints resolves valid annotations and falls back to raw annotations on failure."""
+ assert safe_type_hints(cls, {}) == expected
+
+
+@pytest.mark.parametrize(
+ ("td_cls", "expected_doc", "expected_required"),
+ [
+ pytest.param(
+ SampleTypedDict,
+ "TypedDict doc.",
+ {"req_field": True, "opt_field": False, "plain_opt": False},
+ id="partial-typeddict-with-doc",
+ ),
+ pytest.param(
+ _TotalTypedDict,
+ None,
+ {"a": True},
+ id="total-typeddict-without-doc",
+ ),
+ ],
+)
+def test_typeddict_to_model_variants(
+ td_cls: type[Any],
+ expected_doc: str | None,
+ expected_required: dict[str, bool],
+) -> None:
+ """Verify typeddict_to_model preserves docstrings and per-field requiredness."""
+ model = typeddict_to_model(td_cls, SchemaBuildContext())
+
+ assert model.__doc__ == expected_doc
+ assert {name: f.is_required() for name, f in model.model_fields.items()} == expected_required
+
+
+def test_dataclass_to_model_preserves_doc_and_defaults() -> None:
+ """Verify dataclass_to_model preserves docstrings, required fields, defaults, and default factories."""
+ dc_model = dataclass_to_model(SampleDataclass, SchemaBuildContext())
+
+ assert dc_model.__doc__ == "Dataclass doc.\n\nMore details."
+ assert dc_model.model_fields["req"].is_required() is True
+ assert dc_model.model_fields["with_default"].default == "abc"
+ assert dc_model.model_fields["with_factory"].default_factory is list
+
+
+@pytest.mark.parametrize(
+ ("source_type", "expect_synthesized_model"),
+ [
+ pytest.param(int | str, False, id="union-passthrough"),
+ pytest.param(SampleModel, False, id="model-passthrough"),
+ pytest.param(int, False, id="scalar-passthrough"),
+ pytest.param(SampleTypedDict, True, id="typeddict-synthesized"),
+ pytest.param(SampleDataclass, True, id="dataclass-synthesized"),
+ ],
+)
+def test_coerce_structured_type_to_model_variants(source_type: object, expect_synthesized_model: bool) -> None:
+ """Verify coerce_structured_type_to_model synthesizes models for TypedDict and dataclass only."""
+ result = coerce_structured_type_to_model(source_type, SchemaBuildContext())
+
+ if expect_synthesized_model:
+ assert isinstance(result, type)
+ assert issubclass(result, BaseModel)
+ else:
+ assert result == source_type
diff --git a/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_unit.py b/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_unit.py
new file mode 100644
index 00000000..39683916
--- /dev/null
+++ b/tests/cdm_data_loaders/converters/sources/test_pydantic_reader_unit.py
@@ -0,0 +1,132 @@
+"""Unit tests for cdm_data_loaders.converters.sources.pydantic.reader.PydanticReader."""
+
+from typing import Annotated, Any
+
+import pytest
+from frozendict import frozendict
+from pydantic import BaseModel, Field, RootModel, ValidationError
+
+from cdm_data_loaders.converters.core.errors import ConversionError
+from cdm_data_loaders.converters.core.ir import Provenance
+from cdm_data_loaders.converters.sources.pydantic.reader import PydanticReader
+
+
+class _ReaderConvError(ConversionError):
+ """Custom error for PydanticReader error_by_code tests."""
+
+
+class _ModelA(BaseModel):
+ a_field: int = Field(alias="aAlias")
+
+
+class _ModelB(BaseModel):
+ b_field: str
+
+
+class _RootSingle(RootModel[_ModelA]):
+ """Root single doc."""
+
+
+class _RootUnion(RootModel[_ModelA | _ModelB | None]):
+ pass
+
+
+def _reader() -> PydanticReader:
+ return PydanticReader(by_alias=True, error_by_code=frozendict({"E2": _ReaderConvError, "E7": _ReaderConvError}))
+
+
+def test_pydantic_reader_invalid_by_alias_raises() -> None:
+ """Verify PydanticReader rejects non-boolean by_alias configuration values."""
+ with pytest.raises(ValidationError):
+ PydanticReader(by_alias="yes") # type: ignore[arg-type]
+
+
+@pytest.mark.parametrize(
+ "bad_root",
+ [
+ pytest.param(RootModel[int], id="scalar-root"),
+ pytest.param(RootModel[int | None], id="optional-scalar-root"),
+ pytest.param(RootModel[None], id="none-root"),
+ ],
+)
+def test_pydantic_reader_read_invalid_root_model_raises(bad_root: type[RootModel[Any]]) -> None:
+ """Verify PydanticReader.read rejects RootModel types that do not wrap model objects."""
+ with pytest.raises(_ReaderConvError, match="RootModel"):
+ _reader().read(bad_root)
+
+
+def test_pydantic_reader_read_root_model_single() -> None:
+ """Verify PydanticReader.read lowers a single-model RootModel with aliases, docstrings, and provenance."""
+ doc = _reader().read(_RootSingle)
+
+ assert doc.name == "_RootSingle"
+ assert doc.identifier == f"urn:pydantic:{_RootSingle.__module__}._RootSingle"
+ assert doc.annotations == {"description": "Root single doc."}
+ assert doc.root.annotations == {"description": "Root single doc."}
+ assert doc.root.nullable is False
+ assert doc.root.properties[0].name == "aAlias"
+ assert doc.provenance == Provenance("pydantic", ("_RootSingle",))
+
+
+def test_pydantic_reader_read_root_model_union() -> None:
+ """Verify PydanticReader.read merges fields across a union RootModel."""
+ doc = _reader().read(_RootUnion)
+
+ assert doc.annotations == {}
+ assert doc.root.annotations == {}
+ assert {f.name for f in doc.root.properties} == {"aAlias", "b_field"}
+
+
+def test_pydantic_reader_read_local_model_strips_locals_from_identifier() -> None:
+ """Verify PydanticReader.read omits .. segments from locally defined model identifiers."""
+
+ class _LocalModel(BaseModel):
+ x: int
+
+ doc = _reader().read(_LocalModel)
+
+ assert doc.identifier is not None
+ assert ".." not in doc.identifier
+
+
+@pytest.mark.parametrize(
+ ("annotation", "expected_type", "expected_nullable", "expected_provenance"),
+ [
+ pytest.param(
+ int,
+ "integer",
+ False,
+ Provenance("pydantic", (), {"pydantic_type": "integer"}),
+ id="int",
+ ),
+ pytest.param(
+ int | str | None,
+ "string",
+ True,
+ None,
+ id="optional-scalar-union",
+ ),
+ ],
+)
+def test_pydantic_reader_read_scalar_annotation_valid(
+ annotation: object,
+ expected_type: str,
+ expected_nullable: bool,
+ expected_provenance: Provenance | None,
+) -> None:
+ """Verify read_scalar_annotation resolves scalar types, unions, and nullability."""
+ reader = PydanticReader(error_by_code=frozendict({"E7": _ReaderConvError}))
+ node = reader.read_scalar_annotation(annotation)
+
+ assert node.type == expected_type
+ assert node.nullable is expected_nullable
+ if expected_provenance is not None:
+ assert node.provenance == expected_provenance
+
+
+def test_pydantic_reader_read_scalar_annotation_none_only_raises() -> None:
+ """Verify read_scalar_annotation rejects unions containing only NoneType."""
+ reader = PydanticReader(error_by_code=frozendict({"E7": _ReaderConvError}))
+
+ with pytest.raises(_ReaderConvError, match="only NoneType"):
+ reader.read_scalar_annotation(Annotated[None, "a"] | Annotated[None, "b"])
diff --git a/tests/cdm_data_loaders/converters/sources/test_pydantic_unions.py b/tests/cdm_data_loaders/converters/sources/test_pydantic_unions.py
new file mode 100644
index 00000000..e94df891
--- /dev/null
+++ b/tests/cdm_data_loaders/converters/sources/test_pydantic_unions.py
@@ -0,0 +1,281 @@
+"""Unit tests for Pydantic union and container IR builders in cdm_data_loaders.converters.sources.pydantic.unions."""
+
+from typing import Any
+
+import pytest
+from pydantic import BaseModel, Field
+
+from cdm_data_loaders.converters.core.errors import ConversionError
+from cdm_data_loaders.converters.core.ir import NodeHints, Provenance, TypedNode
+from cdm_data_loaders.converters.sources.pydantic.annotations import unwrap_annotation
+from cdm_data_loaders.converters.sources.pydantic.model_synthesis import SchemaBuildContext
+from cdm_data_loaders.converters.sources.pydantic.unions import (
+ build_field_ir,
+ build_merged_models_struct,
+ build_multi_branch_node,
+ build_parsed_node,
+)
+
+
+class _ModelA(BaseModel):
+ shared: int = Field(description="from A")
+ only_a: str
+
+
+class _ModelB(BaseModel):
+ shared: float
+ only_b: bool
+
+
+class _EmptyModel(BaseModel):
+ pass
+
+
+class _ExplicitValueField(BaseModel):
+ val: str = Field(alias="_value")
+
+
+class _LeafA(BaseModel):
+ a: int
+
+
+class _LeafB(BaseModel):
+ b: str
+
+
+def _ctx(
+ *,
+ max_recursion_depth: int = 3,
+ model_counts: dict[type[BaseModel], int] | None = None,
+ by_alias: bool = False,
+) -> SchemaBuildContext:
+ return SchemaBuildContext(
+ model_counts=model_counts or {},
+ max_recursion_depth=max_recursion_depth,
+ by_alias=by_alias,
+ error_type=ConversionError,
+ error_by_code={},
+ )
+
+
+def test_build_field_ir_sequence() -> None:
+ """Verify build_field_ir lowers sequence annotations with element provenance."""
+ seq_field = build_field_ir("tags", list[str], _ctx(), ("tags",), required_in_parent=True)
+
+ assert seq_field.node.type == "array"
+ assert seq_field.node.nullable is False
+ assert seq_field.node.provenance == Provenance("pydantic", ("tags",))
+ assert seq_field.node.items is not None
+ assert seq_field.node.items.provenance == Provenance("pydantic", ("tags", "element"), {"pydantic_type": "string"})
+
+
+def test_build_field_ir_tuple() -> None:
+ """Verify build_field_ir lowers fixed tuples to closed structs with indexed fields."""
+ tup_field = build_field_ir("pair", tuple[str, int], _ctx(), ("pair",), required_in_parent=True)
+
+ assert tup_field.node.type == "object"
+ assert tup_field.node.required_names == ("item_0", "item_1")
+ assert tup_field.node.additional_properties == TypedNode(type="never", nullable=False)
+ assert tup_field.node.provenance == Provenance("pydantic", ("pair",))
+ assert tup_field.node.properties[0].provenance == Provenance(
+ "pydantic", ("pair", "item_0"), {"pydantic_type": "string"}
+ )
+
+
+def test_build_field_ir_map() -> None:
+ """Verify build_field_ir lowers string-keyed dict annotations to map nodes with key and value provenance."""
+ map_field = build_field_ir("counts", dict[str, int], _ctx(), ("counts",), required_in_parent=False)
+
+ assert map_field.node.type == "map"
+ assert map_field.node.nullable is True
+ assert map_field.node.provenance == Provenance("pydantic", ("counts",))
+ assert map_field.node.key_type == TypedNode(
+ type="string",
+ nullable=False,
+ hints=NodeHints(logical_type="string"),
+ provenance=Provenance("pydantic", ("counts", "key")),
+ )
+ assert map_field.node.value_type is not None
+ assert map_field.node.value_type.provenance == Provenance(
+ "pydantic", ("counts", "value"), {"pydantic_type": "integer"}
+ )
+
+
+@pytest.mark.parametrize(
+ "bad_ann",
+ [
+ pytest.param(list, id="untyped-list"),
+ pytest.param(tuple[()], id="empty-tuple"),
+ pytest.param(dict, id="untyped-dict"),
+ pytest.param(dict[int, str], id="non-str-dict-key"),
+ pytest.param(None, id="none"),
+ ],
+)
+def test_build_field_ir_invalid_container_raises(bad_ann: object) -> None:
+ """Verify build_field_ir raises ConversionError for invalid container annotations."""
+ with pytest.raises(ConversionError):
+ build_field_ir("bad", bad_ann, _ctx(), ("bad",), required_in_parent=True)
+
+
+def test_build_merged_models_struct_merges_two_models() -> None:
+ """Verify build_merged_models_struct intersects required fields and widens shared field types."""
+ merged = build_merged_models_struct([_ModelA, _ModelB], _ctx(), ("u",), nullable=False)
+ by_name = {f.name: f for f in merged.properties}
+
+ assert merged.required_names == ("shared",)
+ assert merged.additional_properties == TypedNode(type="never", nullable=False)
+ assert merged.provenance == Provenance("pydantic", ("u",), {})
+ assert by_name["shared"].annotations == {"description": "from A"}
+ assert by_name["shared"].provenance == Provenance(
+ "pydantic", ("u", "shared"), {"pydantic_type": "integer | number"}
+ )
+ assert by_name["only_a"].required is False
+
+
+def test_build_merged_models_struct_with_scalar_branches() -> None:
+ """Verify build_merged_models_struct synthesizes a nullable _value field for scalar union branches."""
+ scalar_branch, s_ctx = unwrap_annotation(str, _ctx())
+ mixed = build_merged_models_struct([_ModelA], s_ctx, ("m",), nullable=True, scalar_branches=[scalar_branch])
+ by_name = {f.name: f for f in mixed.properties}
+
+ assert mixed.required_names == ()
+ assert by_name["_value"].required is False
+ assert by_name["_value"].node.nullable is True
+ assert by_name["_value"].provenance == Provenance("pydantic", ("m", "_value"), {"pydantic_type": "string"})
+
+
+def test_build_merged_models_struct_inline_struct_on_recursion() -> None:
+ """Verify build_merged_models_struct marks recursive model expansions with inline_struct provenance."""
+ inline_ctx = _ctx(model_counts={_ModelA: 1}, max_recursion_depth=2)
+ inlined = build_merged_models_struct([_ModelA], inline_ctx, ("rec",), nullable=False)
+
+ assert inlined.provenance == Provenance(
+ "pydantic", ("rec",), {"component_id": id(_ModelA), "type_name": "_ModelA", "inline_struct": True}
+ )
+
+
+def test_build_parsed_node_recursion_cutoff() -> None:
+ """Verify build_parsed_node truncates recursive models to a string node at max_recursion_depth."""
+ cutoff_ctx = _ctx(model_counts={_ModelA: 1}, max_recursion_depth=1)
+ parsed_a, p_ctx = unwrap_annotation(_ModelA, cutoff_ctx)
+ cutoff_node = build_parsed_node(parsed_a, p_ctx, ("cut",), nullable=True)
+
+ assert cutoff_node == TypedNode(
+ type="string",
+ nullable=True,
+ hints=NodeHints(logical_type="string"),
+ provenance=Provenance("pydantic", ("cut",)),
+ )
+
+
+@pytest.mark.parametrize(
+ ("models", "by_alias", "with_scalar"),
+ [
+ pytest.param([_EmptyModel], False, False, id="empty-model"),
+ pytest.param([_ExplicitValueField], True, True, id="value-alias-collision"),
+ ],
+)
+def test_build_merged_models_struct_invalid_raises(
+ models: list[type[BaseModel]],
+ by_alias: bool,
+ with_scalar: bool,
+) -> None:
+ """Verify build_merged_models_struct rejects empty models and _value alias collisions."""
+ ctx = _ctx(by_alias=by_alias)
+ scalar_branches = [unwrap_annotation(str, ctx)[0]] if with_scalar else None
+
+ with pytest.raises(ConversionError):
+ build_merged_models_struct(models, ctx, ("err",), nullable=False, scalar_branches=scalar_branches)
+
+
+def test_build_multi_branch_node_matching_containers() -> None:
+ """Verify build_multi_branch_node unifies compatible sequence container branches."""
+ ctx = _ctx()
+ branches = [unwrap_annotation(list[str], ctx), unwrap_annotation(tuple[str, ...], ctx)]
+
+ unified = build_multi_branch_node(branches, ("c",), nullable=False)
+
+ assert unified.type == "array"
+
+
+@pytest.mark.parametrize(
+ ("annotations", "path", "nullable", "expected_type", "expected_provenance"),
+ [
+ pytest.param(
+ (int, None),
+ ("opt_i",),
+ True,
+ "integer",
+ None,
+ id="single-non-null-with-none",
+ ),
+ pytest.param(
+ (int, float),
+ ("num",),
+ False,
+ "number",
+ Provenance("pydantic", ("num",), {"pydantic_type": "integer | number"}),
+ id="pure-scalar-union",
+ ),
+ ],
+)
+def test_build_multi_branch_node_scalar_variants(
+ annotations: tuple[Any, ...],
+ path: tuple[str, ...],
+ nullable: bool,
+ expected_type: str,
+ expected_provenance: Provenance | None,
+) -> None:
+ """Verify build_multi_branch_node lowers optional single-scalar and multi-scalar unions."""
+ ctx = _ctx()
+ branches = [unwrap_annotation(ann, ctx) for ann in annotations]
+
+ node = build_multi_branch_node(branches, path, nullable=nullable)
+
+ assert node.type == expected_type
+ assert node.nullable is nullable
+ if expected_provenance is not None:
+ assert node.provenance == expected_provenance
+
+
+def test_build_multi_branch_node_model_and_scalar() -> None:
+ """Verify build_multi_branch_node merges model and scalar branches into an object with _value."""
+ ctx = _ctx()
+ branches = [unwrap_annotation(_LeafA, ctx), unwrap_annotation(str, ctx)]
+
+ node = build_multi_branch_node(branches, ("mix",), nullable=True)
+
+ assert node.type == "object"
+ assert node.nullable is True
+ assert {f.name for f in node.properties} == {"a", "_value"}
+
+
+def test_build_multi_branch_node_recursion_cutoff() -> None:
+ """Verify build_multi_branch_node falls back to string when all model branches hit recursion cutoff."""
+ rec_ctx = _ctx(model_counts={_LeafA: 1, _LeafB: 1}, max_recursion_depth=1)
+ branches = [unwrap_annotation(_LeafA, rec_ctx), unwrap_annotation(_LeafB, rec_ctx)]
+
+ node = build_multi_branch_node(branches, ("rec_u",), nullable=True)
+
+ assert node == TypedNode(
+ type="string",
+ nullable=True,
+ hints=NodeHints(logical_type="string"),
+ provenance=Provenance("pydantic", ("rec_u",)),
+ )
+
+
+@pytest.mark.parametrize(
+ ("annotations", "nullable"),
+ [
+ pytest.param((list[str], dict[str, int]), False, id="mismatched-containers"),
+ pytest.param((None,), True, id="null-only-union"),
+ ],
+)
+def test_build_multi_branch_node_invalid_raises(annotations: tuple[Any, ...], nullable: bool) -> None:
+ """Verify build_multi_branch_node rejects mismatched container branches and null-only unions."""
+ ctx = _ctx()
+ branches = [unwrap_annotation(ann, ctx) for ann in annotations]
+
+ with pytest.raises(ConversionError):
+ build_multi_branch_node(branches, ("bad",), nullable=nullable)
diff --git a/tests/cdm_data_loaders/converters/sources/test_strict_combinators.py b/tests/cdm_data_loaders/converters/sources/test_strict_combinators.py
new file mode 100644
index 00000000..b943c52a
--- /dev/null
+++ b/tests/cdm_data_loaders/converters/sources/test_strict_combinators.py
@@ -0,0 +1,187 @@
+"""Unit tests for strict JSON Schema combinator merging and node classification."""
+
+from typing import Any, Final
+
+import pytest
+
+from cdm_data_loaders.converters.core.errors import ConversionError
+from cdm_data_loaders.converters.core.extensions import DEFAULT_EXTENSIONS
+from cdm_data_loaders.converters.sources.json_schema.strict_combinators import (
+ classify_node_kind,
+ classify_resolved_node_kind,
+ extract_choice_branches_into_object,
+ extract_object_parts_with_refs,
+ merge_property_definitions,
+ merge_ref_target_and_siblings,
+)
+from cdm_data_loaders.converters.sources.json_schema.strict_refs import StrictBuildContext
+
+SINGLE_PROP_DEF: Final[dict[str, Any]] = {"type": "string"}
+IDENTICAL_PROP_A: Final[dict[str, Any]] = {"type": "integer", "minimum": 1}
+IDENTICAL_PROP_B: Final[dict[str, Any]] = {"minimum": 1, "type": "integer"}
+
+
+def _ctx(defs: dict[str, Any] | None = None) -> StrictBuildContext:
+ return StrictBuildContext(
+ defs=defs or {},
+ ref_counts={},
+ max_recursion_depth=3,
+ extension_registry=DEFAULT_EXTENSIONS,
+ error_type=ConversionError,
+ error_by_code={},
+ )
+
+
+@pytest.mark.parametrize(
+ ("definitions", "expected", "expect_same_identity"),
+ [
+ pytest.param([SINGLE_PROP_DEF], SINGLE_PROP_DEF, True, id="single-definition"),
+ pytest.param([IDENTICAL_PROP_A, IDENTICAL_PROP_B], IDENTICAL_PROP_A, True, id="identical-reordered-keys"),
+ pytest.param(
+ [{"type": "integer"}, {"type": "string", "description": "str branch"}],
+ {
+ "anyOf": [{"type": "integer"}, {"type": "string", "description": "str branch"}],
+ "description": "str branch",
+ },
+ False,
+ id="differing-definitions-wrapped-in-anyof",
+ ),
+ ],
+)
+def test_merge_property_definitions_variants(
+ definitions: list[dict[str, Any]],
+ expected: dict[str, Any],
+ expect_same_identity: bool,
+) -> None:
+ """Verify merge_property_definitions deduplicates identical schemas and wraps differing ones in anyOf."""
+ result = merge_property_definitions(definitions)
+
+ if expect_same_identity:
+ assert result is expected
+ else:
+ assert result == expected
+
+
+def test_merge_ref_target_and_siblings_combines_fields() -> None:
+ """Verify merge_ref_target_and_siblings strips $id/$ref and merges properties, required, and allOf."""
+ target = {
+ "$id": "urn:target",
+ "type": "object",
+ "properties": {"a": {"type": "integer"}, "b": {"type": "string"}},
+ "required": ["a"],
+ "allOf": [{"properties": {"c": {"type": "boolean"}}}],
+ }
+ node = {
+ "$ref": "#/$defs/T",
+ "properties": {"a": {"type": "number"}, "d": {"type": "string"}},
+ "required": ["a", "d"],
+ "allOf": [{"properties": {"e": {"type": "integer"}}}],
+ "description": "merged",
+ }
+
+ merged = merge_ref_target_and_siblings(target, node)
+
+ assert "$id" not in merged
+ assert "$ref" not in merged
+ assert merged["required"] == ["a", "d"]
+ assert merged["description"] == "merged"
+ expected_all_of_len = 2
+ assert len(merged["allOf"]) == expected_all_of_len
+ assert merged["properties"]["a"] == {"anyOf": [{"type": "integer"}, {"type": "number"}]}
+ assert merged["properties"]["d"] == {"type": "string"}
+
+
+@pytest.mark.parametrize(
+ ("schema", "expected_kind"),
+ [
+ pytest.param({"anyOf": [{"type": "string"}]}, "union", id="anyof-union"),
+ pytest.param({"allOf": [{"properties": {"x": {"type": "int"}}}]}, "object", id="allof-object"),
+ pytest.param({"type": ["integer", "number"]}, "scalar", id="multi-scalar-type"),
+ pytest.param(
+ {"type": "object", "additionalProperties": {"type": "string"}},
+ "map",
+ id="additional-properties-map",
+ ),
+ pytest.param(
+ {"type": "object", "properties": {"a": {"type": "string"}}, "additionalProperties": {"type": "string"}},
+ "object",
+ id="properties-and-additional-properties-object",
+ ),
+ pytest.param({"prefixItems": [{"type": "string"}]}, "array", id="prefix-items-array"),
+ ],
+)
+def test_classify_resolved_node_kind_valid(schema: dict[str, Any], expected_kind: str) -> None:
+ """Verify classify_resolved_node_kind classifies union, object, scalar, map, and array schemas."""
+ assert classify_resolved_node_kind(schema, error_type=ConversionError, error_by_code={}) == expected_kind
+
+
+def test_classify_resolved_node_kind_mixed_object_and_scalar_raises() -> None:
+ """Verify classify_resolved_node_kind rejects mixed object and scalar type arrays."""
+ with pytest.raises(ConversionError):
+ classify_resolved_node_kind({"type": ["object", "string"]}, error_type=ConversionError, error_by_code={})
+
+
+def test_classify_node_kind_ref_cycle_raises() -> None:
+ """Verify classify_node_kind raises ConversionError on cyclic $ref definitions."""
+ defs = {"A": {"$ref": "#/$defs/B"}, "B": {"$ref": "#/$defs/A"}}
+
+ with pytest.raises(ConversionError):
+ classify_node_kind(
+ {"$ref": "#/$defs/A"},
+ defs,
+ extension_registry=DEFAULT_EXTENSIONS,
+ error_type=ConversionError,
+ error_by_code={},
+ )
+
+
+def test_extract_object_parts_with_refs_merges_ref_and_conditionals() -> None:
+ """Verify extract_object_parts_with_refs merges $ref targets, dependentSchemas, and then/else branches."""
+ defs = {
+ "Base": {
+ "type": "object",
+ "required": ["id"],
+ "properties": {"id": {"type": "string"}},
+ }
+ }
+
+ props, req, scalars = extract_object_parts_with_refs(
+ {
+ "$ref": "#/$defs/Base",
+ "required": ["extra"],
+ "properties": {"extra": {"type": "integer"}},
+ "dependentSchemas": {"extra": {"properties": {"dep": {"type": "boolean"}}}},
+ "then": {"properties": {"then_prop": {"type": "string"}}},
+ "else": {"properties": {"else_prop": {"type": "string"}}},
+ },
+ _ctx(defs),
+ )
+
+ assert req == {"id", "extra"}
+ assert scalars == []
+ assert set(props) == {"id", "extra", "dep", "then_prop", "else_prop"}
+
+
+def test_extract_object_parts_with_refs_non_object_allof_raises() -> None:
+ """Verify extract_object_parts_with_refs rejects non-object allOf branches."""
+ with pytest.raises(ConversionError):
+ extract_object_parts_with_refs({"allOf": [{"type": "array", "items": {"type": "string"}}]}, _ctx())
+
+
+@pytest.mark.parametrize(
+ ("branches", "keyword"),
+ [
+ pytest.param([{"type": "null"}], "anyOf", id="null-only-anyof"),
+ pytest.param([{"type": "array", "items": {"type": "string"}}], "oneOf", id="array-branch-oneof"),
+ ],
+)
+def test_extract_choice_branches_into_object_invalid_raises(
+ branches: list[dict[str, Any]],
+ keyword: str,
+) -> None:
+ """Verify extract_choice_branches_into_object rejects null-only and array choice branches."""
+ raw_props: dict[str, Any] = {}
+ scalar_list: list[dict[str, Any]] = []
+
+ with pytest.raises(ConversionError):
+ extract_choice_branches_into_object(branches, keyword, _ctx(), raw_props, scalar_list)
diff --git a/tests/cdm_data_loaders/converters/sources/test_strict_json_schema.py b/tests/cdm_data_loaders/converters/sources/test_strict_json_schema.py
new file mode 100644
index 00000000..dd437622
--- /dev/null
+++ b/tests/cdm_data_loaders/converters/sources/test_strict_json_schema.py
@@ -0,0 +1,240 @@
+"""Unit tests for StrictJsonSchemaReader IR construction, provenance, annotations, and error handling."""
+
+from typing import Any, Final
+
+import pytest
+
+from cdm_data_loaders.converters.core.errors import ConversionError
+from cdm_data_loaders.converters.core.ir import Field, NodeHints, Provenance, TypedNode
+from cdm_data_loaders.converters.sources.json_schema.strict_reader import StrictJsonSchemaReader
+
+DIALECT: Final[str] = "https://json-schema.org/draft/2020-12/schema"
+
+
+def test_strict_json_schema_reader_read_node_scalar() -> None:
+ """Verify StrictJsonSchemaReader.read_node lowers a scalar fragment with constraints and provenance."""
+ scalar = StrictJsonSchemaReader().read_node({"type": "integer", "minimum": 1, "description": "pos int"})
+
+ assert scalar.type == "integer"
+ assert scalar.nullable is False
+ assert scalar.annotations == {"description": "pos int"}
+ assert scalar.constraints["minimum"] == 1
+ assert scalar.provenance == Provenance("json-schema", (), {"jsonschema_type": "integer"})
+
+
+def test_strict_json_schema_reader_read_node_array() -> None:
+ """Verify StrictJsonSchemaReader.read_node lowers an array fragment with item provenance."""
+ arr = StrictJsonSchemaReader().read_node({"type": "array", "items": {"type": "string"}, "description": "tags"})
+
+ assert arr.type == "array"
+ assert arr.hints == NodeHints(logical_type="list")
+ assert arr.annotations == {"description": "tags"}
+ assert arr.provenance == Provenance("json-schema", ())
+ assert arr.items is not None
+ assert arr.items.provenance == Provenance("json-schema", ("items",), {"jsonschema_type": "string"})
+
+
+def test_strict_json_schema_reader_read_node_tuple() -> None:
+ """Verify StrictJsonSchemaReader.read_node lowers a closed prefixItems tuple to a struct."""
+ tup = StrictJsonSchemaReader().read_node(
+ {
+ "type": "array",
+ "prefixItems": [{"type": "string"}, {"type": "integer"}],
+ "minItems": 1,
+ "items": False,
+ "description": "pair",
+ }
+ )
+
+ assert tup.type == "object"
+ assert tup.hints == NodeHints(logical_type="struct")
+ assert tup.required_names == ("item_0",)
+ assert tup.additional_properties == TypedNode(type="never", nullable=False)
+ assert tup.annotations == {"description": "pair"}
+ assert tup.provenance == Provenance("json-schema", ())
+ assert tup.properties[0] == Field(
+ "item_0",
+ TypedNode(
+ type="string",
+ declared_type="string",
+ nullable=False,
+ hints=NodeHints(logical_type="string"),
+ provenance=Provenance("json-schema", ("prefixItems", 0), {"jsonschema_type": "string"}),
+ ),
+ required=True,
+ provenance=Provenance("json-schema", ("prefixItems", 0), {"jsonschema_type": "string"}),
+ )
+
+
+def test_strict_json_schema_reader_read_node_map() -> None:
+ """Verify StrictJsonSchemaReader.read_node lowers an object map fragment with key and value provenance."""
+ mp = StrictJsonSchemaReader().read_node(
+ {
+ "type": "object",
+ "additionalProperties": {"type": "integer"},
+ "patternProperties": {"^x_": {"type": "integer"}},
+ "description": "counts",
+ }
+ )
+
+ assert mp.type == "map"
+ assert mp.hints == NodeHints(logical_type="map")
+ assert mp.annotations == {"description": "counts"}
+ assert mp.provenance == Provenance("json-schema", ())
+ assert mp.key_type == TypedNode(
+ type="string",
+ nullable=False,
+ hints=NodeHints(logical_type="string"),
+ provenance=Provenance("json-schema", (), {"jsonschema_type": "string"}),
+ )
+ assert mp.value_type is not None
+ assert mp.value_type.provenance == Provenance(
+ "json-schema", ("additionalProperties",), {"jsonschema_type": "integer"}
+ )
+
+
+def test_strict_json_schema_reader_read_refs_unions_and_cutoff() -> None:
+ """Verify StrictJsonSchemaReader.read lowers $defs refs, sibling constraints, unions, and recursion cutoffs."""
+ expected_min = 5
+ schema: dict[str, Any] = {
+ "$schema": DIALECT,
+ "$id": "urn:test:strict",
+ "description": "root doc",
+ "$defs": {
+ "PosInt": {"type": "integer", "description": "from def"},
+ "Node": {
+ "type": "object",
+ "required": ["label"],
+ "properties": {
+ "label": {"type": "string"},
+ "next": {"$ref": "#/$defs/Node", "description": "child node"},
+ },
+ },
+ },
+ "type": "object",
+ "required": ["n", "u_scalar", "u_mixed"],
+ "properties": {
+ "n": {"$ref": "#/$defs/PosInt", "minimum": expected_min},
+ "u_scalar": {"anyOf": [{"type": "integer"}, {"type": "number"}], "description": "num"},
+ "u_mixed": {
+ "oneOf": [
+ {"type": "string"},
+ {"type": "object", "required": ["k"], "properties": {"k": {"type": "string"}}},
+ ]
+ },
+ "node": {"$ref": "#/$defs/Node"},
+ },
+ }
+ doc = StrictJsonSchemaReader(max_recursion_depth=1).read(schema)
+ assert doc.dialect == DIALECT
+ assert doc.identifier == "urn:test:strict"
+ assert doc.annotations == {"description": "root doc"}
+ assert doc.provenance == Provenance("json-schema")
+
+ by_name = {f.name: f for f in doc.root.properties}
+ assert by_name["n"].node.constraints["minimum"] == expected_min
+ assert by_name["n"].node.annotations == {"description": "from def"}
+
+ assert by_name["u_scalar"].node.type == "number"
+ assert by_name["u_scalar"].node.annotations == {"description": "num"}
+ assert by_name["u_scalar"].node.provenance == Provenance(
+ "json-schema", ("properties", "u_scalar"), {"jsonschema_type": "integer | number"}
+ )
+
+ mixed_fields = {f.name: f for f in by_name["u_mixed"].node.properties}
+ assert mixed_fields["k"].required is False
+ assert mixed_fields["_value"].required is False
+ assert mixed_fields["_value"].provenance == Provenance(
+ "json-schema", ("properties", "u_mixed", "_value"), {"jsonschema_type": "string"}
+ )
+
+ node_props = {f.name: f for f in by_name["node"].node.properties}
+ assert node_props["next"].node.type == "string"
+ assert node_props["next"].node.annotations == {"description": "child node"}
+ assert node_props["next"].node.provenance == Provenance("json-schema", ("properties", "node", "properties", "next"))
+
+
+@pytest.mark.parametrize(
+ "fragment",
+ [
+ pytest.param(
+ {"type": "array", "prefixItems": [{"type": "string"}], "items": {"type": "string"}}, id="open-tuple"
+ ),
+ pytest.param({"type": "array", "prefixItems": [], "items": False}, id="empty-prefixItems"),
+ pytest.param({"anyOf": [{"type": "null"}]}, id="null-only-anyOf"),
+ pytest.param(
+ {"anyOf": [{"type": "string"}, {"type": "array", "items": {"type": "string"}}]},
+ id="incompatible-anyOf-array-scalar",
+ ),
+ pytest.param({"type": "object", "properties": {}}, id="empty-object"),
+ pytest.param(
+ {
+ "oneOf": [
+ {"type": "string"},
+ {"type": "object", "properties": {"_value": {"type": "string"}}},
+ ]
+ },
+ id="value-collision-in-mixed-union",
+ ),
+ ],
+)
+def test_strict_json_schema_reader_read_node_unsupported_raises(fragment: dict[str, Any]) -> None:
+ """Verify StrictJsonSchemaReader.read_node raises ConversionError on unsupported strict constructs."""
+ with pytest.raises(ConversionError):
+ StrictJsonSchemaReader().read_node(fragment)
+
+
+def test_strict_json_schema_reader_read_inline_struct_siblings_and_object_union() -> None:
+ """Verify StrictJsonSchemaReader.read handles inline_struct, object siblings, and multi-object unions."""
+ expected_max = 10
+ expected_min = 2
+ schema: dict[str, Any] = {
+ "$schema": DIALECT,
+ "$defs": {
+ "BoundedInt": {"type": "integer", "maximum": expected_max},
+ "Node": {
+ "type": "object",
+ "required": ["label"],
+ "properties": {
+ "label": {"type": "string"},
+ "next": {"$ref": "#/$defs/Node"},
+ },
+ },
+ "PartA": {"type": "object", "required": ["a"], "properties": {"a": {"type": "integer"}}},
+ "PartB": {"type": "object", "required": ["b"], "properties": {"b": {"type": "string"}}},
+ },
+ "type": "object",
+ "properties": {
+ "bounded": {"$ref": "#/$defs/BoundedInt", "minimum": expected_min},
+ "ext_node": {"$ref": "#/$defs/Node", "properties": {"extra": {"type": "boolean"}}},
+ "node_d2": {"$ref": "#/$defs/Node"},
+ "opt_str": {"anyOf": [{"type": "string"}, {"type": "null"}]},
+ "obj_union": {"oneOf": [{"$ref": "#/$defs/PartA"}, {"$ref": "#/$defs/PartB"}]},
+ },
+ }
+ doc = StrictJsonSchemaReader(max_recursion_depth=2).read(schema)
+ by_name = {f.name: f for f in doc.root.properties}
+
+ assert by_name["bounded"].node.constraints == {"minimum": expected_min, "maximum": expected_max}
+
+ ext_props = {f.name for f in by_name["ext_node"].node.properties}
+ assert ext_props == {"label", "next", "extra"}
+
+ node_lvl1 = by_name["node_d2"].node
+ assert node_lvl1.provenance is not None
+ assert node_lvl1.provenance.metadata["type_name"] == "Node"
+ assert node_lvl1.provenance.metadata["component_id"] == hash(("jsonschema_ref", "Node"))
+ assert "inline_struct" not in node_lvl1.provenance.metadata
+
+ node_lvl2 = next(f for f in node_lvl1.properties if f.name == "next").node
+ assert node_lvl2.type == "object"
+ assert node_lvl2.provenance is not None
+ assert node_lvl2.provenance.metadata["inline_struct"] is True
+
+ assert by_name["opt_str"].node.type == "string"
+ assert by_name["opt_str"].node.nullable is True
+
+ obj_u = by_name["obj_union"].node
+ assert obj_u.type == "object"
+ assert {f.name for f in obj_u.properties} == {"a", "b"}
+ assert obj_u.required_names == ()
diff --git a/tests/cdm_data_loaders/converters/sources/test_xsd.py b/tests/cdm_data_loaders/converters/sources/test_xsd.py
index 841dbbd6..fc8ed414 100644
--- a/tests/cdm_data_loaders/converters/sources/test_xsd.py
+++ b/tests/cdm_data_loaders/converters/sources/test_xsd.py
@@ -3,7 +3,7 @@
import functools
from collections.abc import Callable
from pathlib import Path
-from typing import Final
+from typing import Any, Final
import pyarrow as pa
import pytest
@@ -11,6 +11,7 @@
from pydantic import ValidationError
from pyspark.sql.types import StructType
+from cdm_data_loaders.converters.core.ir import Provenance, TypedNode
from cdm_data_loaders.converters.emitters.dlt import DltEmitter
from cdm_data_loaders.converters.emitters.json_schema.emitter import JsonSchemaEmitter
from cdm_data_loaders.converters.emitters.pyarrow import PyArrowEmitter
@@ -21,7 +22,12 @@
XsdToPyArrowError,
)
from cdm_data_loaders.converters.facades.xsd_to_pyarrow import XsdToPyArrow
-from cdm_data_loaders.converters.sources.xsd import XsdReader
+from cdm_data_loaders.converters.sources.xsd import (
+ XsdReader,
+ _string_ir_node,
+ _wrap_truncated_leaf_ir_field,
+ simple_type_to_ir_node,
+)
from cdm_data_loaders.schemas.uniref.schema import SCHEMAS as UNIREF_SCHEMAS
from cdm_data_loaders.utils.xml_utils import local_name
from cdm_data_loaders.validation.xsd.traversal import (
@@ -308,3 +314,180 @@ def test_xsd_reader_fail_invalid_options(
"""XsdReader validates max_recursion_depth and requires xml_tag when multiple roots exist."""
with pytest.raises(error_type, match=match):
call()
+
+
+IR_HELPERS_XSD: Final[str] = """
+
+
+ List of ints
+
+
+
+ Recursive type doc
+
+
+
+
+
+
+ Root element doc
+
+
+
+
+
+
+
+
+
+"""
+
+READER_IR_XSD: Final[str] = """
+
+
+ List of ints
+
+
+
+ Root element doc
+
+
+
+
+
+
+
+
+
+"""
+
+
+@pytest.fixture
+def ir_helpers_schema(tmp_path: Path) -> xmlschema.XMLSchema:
+ """Compile the XSD schema fixture used by IR helper tests."""
+ xsd_path = tmp_path / "ir_helpers.xsd"
+ xsd_path.write_text(IR_HELPERS_XSD, encoding="utf-8")
+ return xmlschema.XMLSchema(str(xsd_path))
+
+
+@pytest.fixture
+def reader_ir_schema(tmp_path: Path) -> xmlschema.XMLSchema:
+ """Compile the XSD schema fixture used by XsdReader provenance tests."""
+ xsd_path = tmp_path / "reader_ir.xsd"
+ xsd_path.write_text(READER_IR_XSD, encoding="utf-8")
+ return xmlschema.XMLSchema(str(xsd_path))
+
+
+@pytest.mark.parametrize(
+ ("path", "nullable", "metadata", "expected_metadata"),
+ [
+ pytest.param(("a",), True, {"k": "v"}, {"k": "v"}, id="nullable-with-metadata"),
+ pytest.param(("b",), False, None, {}, id="non-nullable-default-metadata"),
+ ],
+)
+def test_string_ir_node_pass(
+ path: tuple[str, ...],
+ *,
+ nullable: bool,
+ metadata: dict[str, Any] | None,
+ expected_metadata: dict[str, Any],
+) -> None:
+ """_string_ir_node preserves nullable, provenance path, and metadata."""
+ node = (
+ _string_ir_node(path, nullable=nullable)
+ if metadata is None
+ else _string_ir_node(path, nullable=nullable, metadata=metadata)
+ )
+ assert node.nullable is nullable
+ assert node.provenance == Provenance("xsd", path, expected_metadata)
+
+
+@pytest.mark.parametrize(
+ ("type_kind", "path", "nullable", "expected_item_type", "expected_xsd_type"),
+ [
+ pytest.param("builtin", ("attr",), False, "string", "xs:NMTOKEN", id="builtin-nmtokens"),
+ pytest.param("custom", ("ints",), True, "integer", "xs:integer", id="custom-int-list"),
+ ],
+)
+def test_simple_type_to_ir_node_pass_list_types(
+ ir_helpers_schema: xmlschema.XMLSchema,
+ type_kind: str,
+ path: tuple[str, ...],
+ *,
+ nullable: bool,
+ expected_item_type: str,
+ expected_xsd_type: str,
+) -> None:
+ """simple_type_to_ir_node maps built-in and custom XSD list types with item provenance."""
+ xsd_type = (
+ ir_helpers_schema.elements["root"].type.attributes["req_attr"].type
+ if type_kind == "builtin"
+ else ir_helpers_schema.types["DocIntList"]
+ )
+ node = simple_type_to_ir_node(xsd_type, path, nullable=nullable)
+ assert node.type == "array"
+ assert node.nullable is nullable
+ assert node.provenance == Provenance("xsd", path)
+ assert node.items is not None
+ assert node.items.type == expected_item_type
+ assert node.items.nullable is True
+ assert node.items.provenance == Provenance("xsd", (*path, "item"), {"xsd_type": expected_xsd_type})
+
+
+@pytest.mark.parametrize(
+ ("is_list", "nullable", "expected_required", "expected_node_type"),
+ [
+ pytest.param(True, True, False, "array", id="list-nullable"),
+ pytest.param(False, False, True, "string", id="scalar-required"),
+ ],
+)
+def test_wrap_truncated_leaf_ir_field_pass(
+ ir_helpers_schema: xmlschema.XMLSchema,
+ *,
+ is_list: bool,
+ nullable: bool,
+ expected_required: bool,
+ expected_node_type: str,
+) -> None:
+ """_wrap_truncated_leaf_ir_field attaches type documentation and truncated_component_id provenance."""
+ rec_elem = ir_helpers_schema.elements["root"].type.content[1]
+ child_elem = rec_elem.type.content[1]
+ expected_meta = {"truncated_component_id": id(child_elem.type)}
+
+ field = _wrap_truncated_leaf_ir_field(child_elem, ("rec", "child"), is_list=is_list, nullable=nullable)
+ assert field.name == "child"
+ assert field.required is expected_required
+ assert field.annotations == {"description": "Recursive type doc"}
+ assert field.provenance == Provenance("xsd", ("rec", "child"), expected_meta)
+ assert field.node.type == expected_node_type
+ assert field.node.nullable is nullable
+ assert field.node.provenance == Provenance("xsd", ("rec", "child"), expected_meta)
+ if is_list:
+ assert field.node.items is not None
+ assert field.node.items.nullable is True
+ assert field.node.items.provenance == Provenance("xsd", ("rec", "child", "item"), expected_meta)
+
+
+def test_xsd_reader_read_pass_annotations_namespace_and_cache(reader_ir_schema: xmlschema.XMLSchema) -> None:
+ """XsdReader.read attaches element annotations, targetNamespace, provenance, and leaves struct_cache empty."""
+ cache: dict[tuple[int, bool], TypedNode] = {}
+ doc = XsdReader(max_recursion_depth=1).read(reader_ir_schema, struct_cache=cache)
+ assert doc.name == "root"
+ assert doc.identifier == "urn:test:ir"
+ assert doc.annotations == {"description": "Root element doc"}
+ assert doc.root.annotations == {"description": "Root element doc"}
+ assert doc.provenance == Provenance("xsd", ("root",))
+ fields_by_name = {f.name: f for f in doc.root.properties}
+ assert fields_by_name["_req_attr"].required is True
+ assert fields_by_name["ints"].annotations == {"description": "List of ints"}
+ assert fields_by_name["ints"].provenance == Provenance("xsd", ("ints",))
+ assert fields_by_name["val"].provenance == Provenance("xsd", ("val",), {"xsd_type": "xs:string"})
+ assert cache == {}
+
+
+def test_xsd_reader_read_at_level_pass_scalar_leaf_provenance(reader_ir_schema: xmlschema.XMLSchema) -> None:
+ """XsdReader.read_at_level marks scalar leaf tables with scalar_leaf_table provenance and #text property."""
+ level_docs = XsdReader().read_at_level(reader_ir_schema, 1, root_tag="root")
+ assert "val" in level_docs
+ assert level_docs["val"].root.provenance == Provenance("xsd", (), {"scalar_leaf_table": True})
+ assert level_docs["val"].root.properties[0].name == "#text"
diff --git a/tests/cdm_data_loaders/core/test_buffer.py b/tests/cdm_data_loaders/core/test_buffer.py
index 74120e9a..5e8cf66e 100644
--- a/tests/cdm_data_loaders/core/test_buffer.py
+++ b/tests/cdm_data_loaders/core/test_buffer.py
@@ -1,6 +1,9 @@
"""Tests for the page buffer in cdm_data_loaders.core.buffer."""
+import json
from collections.abc import Iterator
+from pathlib import Path
+from types import SimpleNamespace
from typing import Any, Final
import dlt
@@ -15,6 +18,7 @@
DictBuffer,
format_rejected_records,
make_rejected_row,
+ make_rejected_table_result,
resolve_destination_schema,
)
from cdm_data_loaders.core.fields import REJECTED_TABLE_NAME
@@ -447,18 +451,67 @@ def test_resolve_destination_schema_resolves_rejected_and_mapped_tables(
assert resolve_destination_schema(table_name, schema_map) == expected_schema
-@pytest.mark.parametrize("use_pyarrow", [True, False], ids=["pyarrow-page", "dict-list-page"])
-def test_format_rejected_records_formats_pyarrow_and_dict_pages(use_pyarrow: bool) -> None:
+@pytest.mark.parametrize(
+ "use_pyarrow",
+ [
+ pytest.param(None, id="default-pyarrow-page"),
+ pytest.param(True, id="pyarrow-page"),
+ pytest.param(False, id="dict-list-page"),
+ ],
+)
+def test_format_rejected_records_formats_pyarrow_and_dict_pages(use_pyarrow: bool | None) -> None:
"""format_rejected_records wraps rejected rows with _rejected table metadata as PyArrow or list pages."""
rows = [
make_rejected_row("people", "bad row", source_file="people.csv", line_no=2, raw_record="x"),
]
- item = format_rejected_records(rows, use_pyarrow=use_pyarrow)
+ item = (
+ format_rejected_records(rows) if use_pyarrow is None else format_rejected_records(rows, use_pyarrow=use_pyarrow)
+ )
assert item.meta.table_name == REJECTED_TABLE_NAME
- if use_pyarrow:
+ if use_pyarrow is False:
+ assert item.data == rows
+ else:
assert isinstance(item.data, pa.Table)
assert item.data.schema.equals(REJECTED_PYARROW_SCHEMA)
assert item.data.to_pylist() == rows
- else:
- assert item.data == rows
+
+
+def test_make_rejected_table_result_serializes_non_json_values() -> None:
+ """make_rejected_table_result serializes non-JSON values via str and wraps the row under _rejected."""
+ expected_line_no = 42
+ result = make_rejected_table_result(
+ "people",
+ {"p": Path("a.txt")},
+ "bad",
+ source_file="f.jsonl",
+ line_no=expected_line_no,
+ )
+
+ assert result == {
+ REJECTED_TABLE_NAME: [
+ {
+ "table_name": "people",
+ "source_file": "f.jsonl",
+ "line_no": expected_line_no,
+ "raw_record": json.dumps({"p": "a.txt"}),
+ "error_detail": "bad",
+ }
+ ]
+ }
+
+
+def test_dict_buffer_from_settings_preserves_subclass_and_defaults() -> None:
+ """DictBuffer.from_settings instantiates the subclass and defaults missing optional settings."""
+
+ class _SubDictBuffer(DictBuffer):
+ pass
+
+ expected_max_items = 5
+ minimal_settings = SimpleNamespace(buffer_size=expected_max_items)
+ sub_buf = _SubDictBuffer.from_settings(minimal_settings) # type: ignore[arg-type]
+
+ assert isinstance(sub_buf, _SubDictBuffer)
+ assert sub_buf.max_items == expected_max_items
+ assert sub_buf.schema_map is None
+ assert sub_buf.use_pyarrow is True
diff --git a/tests/cdm_data_loaders/core/test_fields.py b/tests/cdm_data_loaders/core/test_fields.py
index 51b4e4d9..655ea186 100644
--- a/tests/cdm_data_loaders/core/test_fields.py
+++ b/tests/cdm_data_loaders/core/test_fields.py
@@ -166,7 +166,7 @@ class TableNameModel(BaseModel):
def test_table_name_fail_reserved_rejected_table_name() -> None:
"""TableName rejects the reserved _rejected table name."""
assert REJECTED_TABLE_NAME == "_rejected"
- with pytest.raises(ValidationError, match=REJECTED_TABLE_NAME):
+ with pytest.raises(ValidationError, match=f"Table name {REJECTED_TABLE_NAME!r} is reserved"):
TableNameModel.model_validate({"table_name": REJECTED_TABLE_NAME})
@@ -262,6 +262,11 @@ def test_file_glob_fail_empty_string() -> None:
f"Glob pattern for XML files. {CTS_GLOB_WILDCARDS_DESCRIPTION}",
id="with_trailing_period",
),
+ pytest.param(
+ "Glob pattern for suffix X.",
+ f"Glob pattern for suffix X. {CTS_GLOB_WILDCARDS_DESCRIPTION}",
+ id="preserves_non_period_trailing_chars",
+ ),
],
)
def test_file_glob_description_pass(summary: str, expected: str) -> None:
diff --git a/tests/cdm_data_loaders/core/test_file_items.py b/tests/cdm_data_loaders/core/test_file_items.py
index 36d3929a..51534f46 100644
--- a/tests/cdm_data_loaders/core/test_file_items.py
+++ b/tests/cdm_data_loaders/core/test_file_items.py
@@ -1,8 +1,11 @@
"""Tests for the FileItemDict helpers in cdm_data_loaders.core.file_items."""
+import errno
import gzip
import io
+import os
from pathlib import Path
+from types import SimpleNamespace
from typing import Final
import fsspec
@@ -95,14 +98,20 @@ def test_as_file_item_fail_missing_file(tmp_path: Path) -> None:
with pytest.raises(FileNotFoundError, match="No such file or directory") as exc_info:
as_file_item(missing)
+ assert exc_info.value.errno == errno.ENOENT
+ assert exc_info.value.strerror == os.strerror(errno.ENOENT)
assert exc_info.value.filename == str(missing)
def test_as_file_item_fail_directory(tmp_path: Path) -> None:
- """Verify a directory raises IsADirectoryError."""
- with pytest.raises(IsADirectoryError):
+ """Verify a directory raises IsADirectoryError with errno, strerror, and filename."""
+ with pytest.raises(IsADirectoryError) as exc_info:
as_file_item(tmp_path)
+ assert exc_info.value.errno == errno.EISDIR
+ assert exc_info.value.strerror == os.strerror(errno.EISDIR)
+ assert exc_info.value.filename == str(tmp_path)
+
"""join_url"""
@@ -169,6 +178,24 @@ def test_source_url_pass_file_item(fs_backend: FsBackend) -> None:
assert source_url(item) == item["file_url"]
+@pytest.mark.parametrize(
+ "named_handle",
+ [
+ pytest.param(True, id="named-file-handle"),
+ pytest.param(False, id="unnamed-bytes-io"),
+ ],
+)
+def test_source_url_pass_open_handle(tmp_path: Path, named_handle: bool) -> None:
+ """Verify source_url uses .name on a file handle and falls back to str(handle) when unnamed."""
+ if named_handle:
+ path = _write(tmp_path, "open.tsv", CONTENT)
+ with path.open("rb") as fh:
+ assert source_url(fh) == str(path)
+ else:
+ buf = io.BytesIO(CONTENT)
+ assert source_url(buf) == str(buf)
+
+
"""open_binary"""
@@ -217,6 +244,16 @@ def test_is_local_pass_matches_backend(fs_backend: FsBackend) -> None:
assert is_local(fs_backend.file_item("data.tsv")) is fs_backend.is_local
+def test_is_local_pass_string_protocol() -> None:
+ """Verify is_local handles filesystems whose protocol attribute is a plain string."""
+ item = FileItemDict(
+ {"file_name": "a.tsv", "relative_path": "a.tsv", "file_url": "file:///a.tsv", "mime_type": "text/tsv"},
+ SimpleNamespace(protocol="file"), # type: ignore[arg-type]
+ )
+
+ assert is_local(item) is True
+
+
"""local_copy"""
diff --git a/tests/cdm_data_loaders/core/test_filesystem.py b/tests/cdm_data_loaders/core/test_filesystem.py
index c94ebccd..78766f50 100644
--- a/tests/cdm_data_loaders/core/test_filesystem.py
+++ b/tests/cdm_data_loaders/core/test_filesystem.py
@@ -294,7 +294,7 @@ def test_collect_file_listing_fail_without_filesystem_resource(
)
with pytest.raises(
ValueError,
- match="loader_file_format='file_listing' requires at least one dlt filesystem resource",
+ match=r"^loader_file_format='file_listing' requires at least one dlt filesystem resource\.$",
):
collect_file_listing(settings, invalid_resource)
@@ -367,6 +367,22 @@ def read_none(_items: TDataItems, _cfg: InputLocalOutputSettings) -> None:
assert collector.preprocess_errors == []
+def test_attach_file_collector_pass_repeated_attach_on_plain_filesystem_rebinds_tracker(tmp_path: Path) -> None:
+ """Calling attach_file_collector twice on a plain filesystem resource rebinds the tracker without double-mapping."""
+ (tmp_path / "only.txt").write_text("hello", encoding="utf-8")
+ fs_resource = filesystem(bucket_url=str(tmp_path), file_glob="*.txt")
+
+ first_collector = attach_file_collector(fs_resource)
+ second_collector = attach_file_collector(fs_resource)
+
+ items = list(fs_resource)
+ assert [item["relative_path"] for item in items] == ["only.txt"]
+ assert first_collector.ingested_files == []
+ assert first_collector.preprocess_errors == []
+ assert second_collector.ingested_files == ["only.txt"]
+ assert second_collector.preprocess_errors == []
+
+
def test_build_filesystem_resource_pass_default_glob_and_callable_transformer(
input_dir: Path, output_dir: Path
) -> None:
@@ -842,3 +858,28 @@ class PresetIngestSettings(FileFormatIngestSettings, cli_prog_name="preset_inges
},
"pipeline_run_kwargs": {"loader_file_format": "parquet"},
}
+
+
+def test_build_file_ingest_kwargs_defaults_fallback_dataset_name_to_false(tmp_path: Path) -> None:
+ """build_file_ingest_kwargs defaults fallback_dataset_name to False and preserves dataset_name=None."""
+ settings = make_cts_settings(
+ FileFormatIngestSettings,
+ tmp_path,
+ dataset_name="unused_ds",
+ loader_file_format="parquet",
+ ).model_copy(update={"dataset_name": None})
+
+ result = build_file_ingest_kwargs(settings, TINY_PIPELINE_NAME)
+
+ assert result == {
+ "pipeline_kwargs": {
+ "pipeline_name": TINY_PIPELINE_NAME,
+ "dataset_name": None,
+ },
+ "pipeline_run_kwargs": {"loader_file_format": "parquet"},
+ }
+
+
+def test_collect_relative_paths_flattens_batched_lists() -> None:
+ """_collect_relative_paths extracts relative_path values from list-batched file items."""
+ assert _collect_relative_paths([[{"relative_path": "a.txt"}, {"relative_path": "b.txt"}]]) == ["a.txt", "b.txt"]
diff --git a/tests/cdm_data_loaders/core/test_md5_preprocessor.py b/tests/cdm_data_loaders/core/test_md5_preprocessor.py
index ecdaef0c..7801cbdd 100644
--- a/tests/cdm_data_loaders/core/test_md5_preprocessor.py
+++ b/tests/cdm_data_loaders/core/test_md5_preprocessor.py
@@ -1,18 +1,21 @@
"""Unit tests for ``cdm_data_loaders.core.md5_preprocessor``."""
import gzip
+from pathlib import Path
from typing import Final
from unittest.mock import patch
import pytest
+from cdm_data_loaders.core import md5_preprocessor
+from cdm_data_loaders.core.file_items import as_file_item
from cdm_data_loaders.core.md5_preprocessor import (
FILE_PREPROCESSORS,
parse_md5_sidecar,
validate_md5_sidecar,
)
from cdm_data_loaders.core.preprocessors import FilePreprocessError
-from cdm_data_loaders.utils.checksums import new_hasher
+from cdm_data_loaders.utils.checksums import HASH_CHUNK_BYTES, new_hasher
from tests.helpers.fs_backends import FsBackend
SAMPLE_DIGEST: Final[str] = "d41d8cd98f00b204e9800998ecf8427e"
@@ -150,3 +153,27 @@ def test_validate_md5_sidecar_fail_oserror_reading_sidecar(fs_backend: FsBackend
),
):
validate_md5_sidecar(file_item)
+
+
+def test_validate_md5_sidecar_forwards_open_and_checksum_arguments(tmp_path: Path) -> None:
+ """validate_md5_sidecar opens sidecar as UTF-8 text, raw stream binary uncompressed, and passes HASH_CHUNK_BYTES."""
+ gz_path = tmp_path / "sample.xml.gz"
+ raw = gzip.compress(b"", mtime=0)
+ gz_path.write_bytes(raw)
+ (tmp_path / "sample.xml.gz.md5").write_text(f"{_md5_hex(raw)} sample.xml.gz\n", encoding="utf-8")
+ file_item = as_file_item(gz_path)
+
+ with (
+ patch.object(file_item.fsspec, "open", wraps=file_item.fsspec.open) as mock_fs_open,
+ patch.object(file_item, "open", wraps=file_item.open) as mock_item_open,
+ patch.object(
+ md5_preprocessor,
+ "compute_stream_checksum",
+ wraps=md5_preprocessor.compute_stream_checksum,
+ ) as mock_checksum,
+ ):
+ validate_md5_sidecar(file_item)
+
+ assert mock_fs_open.call_args_list[0].kwargs == {"mode": "rt", "encoding": "utf-8"}
+ assert mock_item_open.call_args.kwargs == {"mode": "rb", "compression": "disable"}
+ assert mock_checksum.call_args.kwargs["chunk_size"] == HASH_CHUNK_BYTES
diff --git a/tests/cdm_data_loaders/core/test_pipeline_catalog.py b/tests/cdm_data_loaders/core/test_pipeline_catalog.py
index 34af98c0..81f22f57 100644
--- a/tests/cdm_data_loaders/core/test_pipeline_catalog.py
+++ b/tests/cdm_data_loaders/core/test_pipeline_catalog.py
@@ -1,16 +1,23 @@
"""Unit tests for ``cdm_data_loaders.core.pipeline_catalog``."""
+import re
+import sys
from pathlib import Path
from types import ModuleType, SimpleNamespace
from unittest.mock import patch
import pytest
+from pydantic_settings import SettingsConfigDict
from cdm_data_loaders.core import pipeline_catalog
from cdm_data_loaders.core.cli import CliRunner
from cdm_data_loaders.core.pipeline_catalog import (
PACKAGE_DIST_NAME,
PipelineSpec,
+ _build_module_to_cli_map,
+ _resolve_callable_spec,
+ _resolve_runner_spec,
+ _spec_from_module,
discover_pipeline_targets,
load_project_scripts,
resolve_pipeline,
@@ -126,3 +133,325 @@ def test_resolve_pipeline_short_name_and_unregistered_runner(
)
assert resolve_pipeline(target, pyproject_path=pyproject_path) == expected_spec
+
+
+@pytest.mark.parametrize(
+ ("script_spec", "expected_cli_command"),
+ [
+ pytest.param(
+ "cdm_data_loaders.pipelines.jsonlines.extract:cli",
+ "jsonlines_extract",
+ id="registered-with-attr",
+ ),
+ pytest.param(
+ "cdm_data_loaders.pipelines.jsonlines.extract",
+ "jsonlines_extract",
+ id="registered-without-attr",
+ ),
+ pytest.param(
+ "cdm_data_loaders.pipelines.jsonlines.extract:cli",
+ "custom_extract",
+ id="registered-custom-command",
+ ),
+ ],
+)
+def test_resolve_callable_spec_registered_module(
+ script_spec: str,
+ expected_cli_command: str,
+) -> None:
+ """Resolve a callable whose module is registered in module_to_cli."""
+ module_path = "cdm_data_loaders.pipelines.jsonlines.extract"
+ scripts = {expected_cli_command: script_spec}
+ module_to_cli = {module_path: expected_cli_command}
+ expected_spec = PipelineSpec(
+ cli_command=expected_cli_command,
+ module_path=module_path,
+ settings_cls=JsonlExtractSettings,
+ pipeline_fn=run_jsonlines_pipeline,
+ cli_callable=jsonl_extract_cli,
+ )
+
+ assert _resolve_callable_spec(run_jsonlines_pipeline, scripts, module_to_cli) == expected_spec
+
+
+def test_resolve_callable_spec_imports_unloaded_module(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Import the callable's module when it is not yet in sys.modules."""
+ module_path = "cdm_data_loaders.pipelines.jsonlines.extract"
+ original_mod = sys.modules.get(module_path)
+ monkeypatch.delitem(sys.modules, module_path, raising=False)
+ scripts = {"jsonlines_extract": f"{module_path}:cli"}
+ module_to_cli = {module_path: "jsonlines_extract"}
+
+ result = _resolve_callable_spec(run_jsonlines_pipeline, scripts, module_to_cli)
+ reloaded_mod = sys.modules[module_path]
+
+ assert reloaded_mod is not original_mod
+ assert result == PipelineSpec(
+ cli_command="jsonlines_extract",
+ module_path=module_path,
+ settings_cls=reloaded_mod.JsonlExtractSettings,
+ pipeline_fn=reloaded_mod.run_jsonlines_pipeline,
+ cli_callable=reloaded_mod.cli,
+ )
+
+
+def test_resolve_callable_spec_unregistered_module_cli_runner(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Fall back to module.cli when module_path is unregistered and cli.action_fn matches."""
+ mod_name = "tests_synthetic_unregistered_runner_mod"
+
+ def action_fn(_settings: JsonlExtractSettings) -> None:
+ return None
+
+ action_fn.__module__ = mod_name
+ runner = CliRunner(JsonlExtractSettings, action_fn)
+ mod = ModuleType(mod_name)
+ mod.cli = runner # type: ignore[attr-defined]
+ monkeypatch.setitem(sys.modules, mod_name, mod)
+
+ result = _resolve_callable_spec(action_fn, scripts={}, module_to_cli={})
+
+ assert result == PipelineSpec(
+ cli_command="jsonlines_extract",
+ module_path=mod_name,
+ settings_cls=JsonlExtractSettings,
+ pipeline_fn=action_fn,
+ cli_callable=runner,
+ )
+
+
+@pytest.mark.parametrize(
+ "case",
+ ["no_module", "no_cli_attr", "non_runner_cli_attr", "different_action_fn"],
+ ids=["no_module", "no_cli_attr", "non_runner_cli_attr", "different_action_fn"],
+)
+def test_resolve_callable_spec_rejects_unresolvable_callable(
+ monkeypatch: pytest.MonkeyPatch,
+ case: str,
+) -> None:
+ """Raise ValueError with callable and module context when a callable cannot be resolved."""
+ if case == "no_module":
+
+ def no_mod_fn(_settings: JsonlExtractSettings) -> None:
+ return None
+
+ no_mod_fn.__module__ = ""
+ fn: object = no_mod_fn
+ expected_mod = ""
+ else:
+ mod_name = f"tests_synthetic_unresolvable_{case}"
+
+ def target_fn(_settings: JsonlExtractSettings) -> None:
+ return None
+
+ def other_fn(_settings: JsonlExtractSettings) -> None:
+ return None
+
+ target_fn.__module__ = mod_name
+ mod = ModuleType(mod_name)
+ if case == "non_runner_cli_attr":
+ mod.cli = other_fn # type: ignore[attr-defined]
+ elif case == "different_action_fn":
+ mod.cli = CliRunner(JsonlExtractSettings, other_fn) # type: ignore[attr-defined]
+ monkeypatch.setitem(sys.modules, mod_name, mod)
+ fn = target_fn
+ expected_mod = mod_name
+
+ expected_msg = f"Cannot resolve pipeline from callable {fn!r} (module {expected_mod!r})"
+ with pytest.raises(ValueError, match=re.escape(expected_msg)):
+ _resolve_callable_spec(fn, scripts={}, module_to_cli={}) # type: ignore[arg-type]
+
+
+def test_load_project_scripts_defaults_to_repo_pyproject() -> None:
+ """Read repository pyproject.toml when pyproject_path is None."""
+ default_scripts = load_project_scripts()
+ assert "xsv" in default_scripts
+
+
+def test_load_project_scripts_returns_empty_when_scripts_missing(tmp_path: Path) -> None:
+ """Return an empty mapping when [project.scripts] is absent from pyproject.toml."""
+ empty_toml = tmp_path / "empty.toml"
+ empty_toml.write_text("[project]\nname = 'demo'\n", encoding="utf-8")
+
+ assert load_project_scripts(empty_toml) == {}
+ assert discover_pipeline_targets(empty_toml) == {}
+
+
+def test_build_module_to_cli_map_prefers_cli_attr_and_handles_no_colon() -> None:
+ """Prefer :cli over alphabetically earlier aliases and handle specs without a colon."""
+ scripts = {
+ "aaa_alias": "pkg.mod:other",
+ "zzz_primary": "pkg.mod:cli",
+ "bare": "pkg.bare",
+ }
+ assert _build_module_to_cli_map(scripts) == {
+ "pkg.mod": "zzz_primary",
+ "pkg.bare": "bare",
+ }
+
+
+@pytest.mark.parametrize(
+ "case",
+ [
+ pytest.param("explicit_runner", id="explicit-runner-attr"),
+ pytest.param("prefixed_plain_callable", id="command-prefix-plain-callable"),
+ pytest.param("non_callable_attr", id="non-callable-attr"),
+ ],
+)
+def test_spec_from_module_resolves_cli_attribute_variants(case: str) -> None:
+ """Resolve explicit attr_name, _cli fallback, and non-callable attributes on a module."""
+ mod = ModuleType("tests_synthetic_spec_mod")
+ runner = CliRunner(JsonlExtractSettings, run_jsonlines_pipeline)
+
+ def plain_cli() -> None:
+ return None
+
+ mod.custom_cli = runner # type: ignore[attr-defined]
+ mod.plain_cli = plain_cli # type: ignore[attr-defined]
+ mod.not_callable = 42 # type: ignore[attr-defined]
+ mod.Settings = JsonlExtractSettings # type: ignore[attr-defined]
+ mod.run_pipeline = run_jsonlines_pipeline # type: ignore[attr-defined]
+
+ if case == "explicit_runner":
+ result = _spec_from_module("ignored", mod, attr_name="custom_cli")
+ expected = PipelineSpec(
+ cli_command="ignored",
+ module_path="tests_synthetic_spec_mod",
+ settings_cls=JsonlExtractSettings,
+ pipeline_fn=run_jsonlines_pipeline,
+ cli_callable=runner,
+ )
+ elif case == "prefixed_plain_callable":
+ result = _spec_from_module("plain", mod)
+ expected = PipelineSpec(
+ cli_command="plain",
+ module_path="tests_synthetic_spec_mod",
+ settings_cls=JsonlExtractSettings,
+ pipeline_fn=run_jsonlines_pipeline,
+ cli_callable=plain_cli,
+ )
+ else:
+ result = _spec_from_module("other", mod, attr_name="not_callable")
+ expected = PipelineSpec(
+ cli_command="other",
+ module_path="tests_synthetic_spec_mod",
+ settings_cls=JsonlExtractSettings,
+ pipeline_fn=run_jsonlines_pipeline,
+ cli_callable=None,
+ )
+
+ assert result == expected
+
+
+def test_resolve_runner_spec_falls_back_to_module_stem_without_cli_prog_name() -> None:
+ """Derive cli_command from the action module stem when cli_prog_name is None."""
+
+ class NoProgSettings(JsonlExtractSettings):
+ model_config = SettingsConfigDict(**{**JsonlExtractSettings.model_config, "cli_prog_name": None})
+
+ def dummy_action(_s: NoProgSettings) -> None:
+ return None
+
+ dummy_action.__module__ = "pkg.sub.my_pipe"
+ no_prog_runner = CliRunner(NoProgSettings, dummy_action)
+
+ result = _resolve_runner_spec(no_prog_runner, scripts={}, module_to_cli={})
+
+ assert result == PipelineSpec(
+ cli_command="my_pipe",
+ module_path="pkg.sub.my_pipe",
+ settings_cls=NoProgSettings,
+ pipeline_fn=dummy_action,
+ cli_callable=no_prog_runner,
+ )
+
+
+@pytest.mark.parametrize(
+ "target_case",
+ [
+ pytest.param("registered_runner", id="registered-cli-runner"),
+ pytest.param("registered_module", id="registered-module"),
+ pytest.param("unregistered_module_with_cli", id="unregistered-module-with-cli"),
+ pytest.param("registered_callable", id="registered-callable"),
+ ],
+)
+def test_resolve_pipeline_registered_and_fallback_targets(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ target_case: str,
+) -> None:
+ """Resolve registered CliRunner, module, callable, and cli-bearing unregistered module targets."""
+ pyproject = tmp_path / "pyproject.toml"
+ pyproject.write_text(
+ '[project.scripts]\njsonlines_extract = "cdm_data_loaders.pipelines.jsonlines.extract:cli"\n',
+ encoding="utf-8",
+ )
+ expected_module_path = "cdm_data_loaders.pipelines.jsonlines.extract"
+
+ if target_case == "registered_runner":
+ target: object = jsonl_extract_cli
+ elif target_case == "registered_module":
+ target = sys.modules[expected_module_path]
+ elif target_case == "registered_callable":
+ target = run_jsonlines_pipeline
+ else:
+ mod_with_cli = ModuleType("cdm_data_loaders.pipelines.custom.entry")
+ mod_with_cli.cli = jsonl_extract_cli # type: ignore[attr-defined]
+ monkeypatch.setitem(sys.modules, mod_with_cli.__name__, mod_with_cli)
+ target = mod_with_cli
+ expected_module_path = mod_with_cli.__name__
+
+ result = resolve_pipeline(target, pyproject_path=pyproject) # type: ignore[arg-type]
+
+ assert result == PipelineSpec(
+ cli_command="jsonlines_extract",
+ module_path=expected_module_path,
+ settings_cls=JsonlExtractSettings,
+ pipeline_fn=run_jsonlines_pipeline,
+ cli_callable=jsonl_extract_cli,
+ )
+
+
+def test_resolve_pipeline_returns_pipeline_spec_unchanged() -> None:
+ """Return an existing PipelineSpec instance unchanged."""
+ spec = PipelineSpec(cli_command="demo", module_path="pkg.demo")
+ assert resolve_pipeline(spec) is spec
+
+
+@pytest.mark.parametrize(
+ "target",
+ [
+ pytest.param(" jsonlines_extract ", id="whitespace-padded"),
+ pytest.param("cdm_data_loaders.pipelines.jsonlines_extract", id="prefixed-command"),
+ pytest.param("cdm_data_loaders.pipelines.jsonlines.extract", id="full-module-path"),
+ ],
+)
+def test_resolve_pipeline_resolves_string_variants(target: str) -> None:
+ """Resolve whitespace-padded, package-prefixed, and full module path string targets."""
+ result = resolve_pipeline(target)
+ assert result.cli_command == "jsonlines_extract"
+
+
+@pytest.mark.parametrize(
+ ("target", "expected_match"),
+ [
+ pytest.param("nope", r"Unknown pipeline 'nope'\. Known pipelines: alpha, beta", id="unknown-string"),
+ pytest.param(123, r"Unknown pipeline 123\. Known pipelines: alpha, beta", id="unsupported-type"),
+ ],
+)
+def test_resolve_pipeline_fail_unknown_target(
+ tmp_path: Path,
+ target: object,
+ expected_match: str,
+) -> None:
+ """Raise ValueError listing known pipelines for unknown or unsupported targets."""
+ pyproject = tmp_path / "pyproject.toml"
+ pyproject.write_text(
+ '[project.scripts]\nbeta = "pkg.b:cli"\nalpha = "pkg.a:cli"\n',
+ encoding="utf-8",
+ )
+ with pytest.raises(ValueError, match=expected_match):
+ resolve_pipeline(target, pyproject_path=pyproject) # type: ignore[arg-type]
diff --git a/tests/cdm_data_loaders/core/test_reporting.py b/tests/cdm_data_loaders/core/test_reporting.py
index 1108997c..ce0edb16 100644
--- a/tests/cdm_data_loaders/core/test_reporting.py
+++ b/tests/cdm_data_loaders/core/test_reporting.py
@@ -20,6 +20,7 @@
ParsedLoadJobFileName,
TPackageJobState,
)
+from dlt.extract.resource import DltResource
from pydantic import ValidationError
from requests import RequestException
@@ -1116,3 +1117,43 @@ def test_write_pipeline_report_pass_datetime_subclass_without_global_safe_dumper
"pipeline_load_info_errs": None,
},
)
+
+
+def test_save_pipeline_metadata_sets_text_columns_on_load_info_resource(tmp_path: Path) -> None:
+ """save_pipeline_metadata configures text data_type on nullable metadata columns of _load_info."""
+ captured_resources: list[DltResource] = []
+
+ class _PipelineStub:
+ pipeline_name = TEST_PIPELINE_NAME
+ dataset_name = TEST_DATASET_NAME
+ last_trace = SimpleNamespace(started_at=MOCK_START_TIME, finished_at=MOCK_FINISH_TIME)
+
+ def run(self, resource: DltResource, **_kwargs: object) -> SimpleNamespace:
+ captured_resources.append(resource)
+ return SimpleNamespace(has_failed_jobs=False)
+
+ in_dir = tmp_path / "in"
+ in_dir.mkdir()
+ settings = CtsSettings(
+ input_dir=str(in_dir),
+ output_dir=tmp_path / "out",
+ use_destination="local_fs",
+ )
+ (tmp_path / "out" / PIPELINE_DIR_NAME).mkdir(parents=True, exist_ok=True)
+ load_info = create_load_info_one_completed_two_failed()
+
+ save_pipeline_metadata(settings, _PipelineStub(), load_info, err_msg=None) # type: ignore[arg-type]
+
+ assert len(captured_resources) == 1
+ resource = captured_resources[0]
+ assert resource.max_table_nesting == 0
+ assert resource.columns == {
+ k: {"name": k, "data_type": "text"}
+ for k in [
+ "environment",
+ "staging_type",
+ "staging_name",
+ "staging_displayable_credentials",
+ "dataset_name",
+ ]
+ }
diff --git a/tests/cdm_data_loaders/core/test_settings_check_aliases.py b/tests/cdm_data_loaders/core/test_settings_check_aliases.py
index 9a0b50cc..daa8f706 100644
--- a/tests/cdm_data_loaders/core/test_settings_check_aliases.py
+++ b/tests/cdm_data_loaders/core/test_settings_check_aliases.py
@@ -113,6 +113,12 @@ def test_check_aliases_pass_valid_names(
"MySettingsClass: CLI name 'beta' is claimed by both 'alpha' and 'beta'",
id="alias_is_other_field_name",
),
+ pytest.param(
+ {"alpha": Field("a", alias="beta", validation_alias="gamma"), "beta": "b"},
+ None,
+ "MySettingsClass: CLI name 'beta' is claimed by both 'alpha' and 'beta'",
+ id="alias_distinct_from_validation_alias_collides",
+ ),
pytest.param(
{"output_dir": "x"},
{"output-dir": "output-dir"},
@@ -151,8 +157,8 @@ def test_check_aliases_pass_valid_names(
),
pytest.param(
{"alpha": "a"},
- {"alpha": ["-a", "h"], "nope": "n"},
- "MySettingsClass: invalid shortcut '-a' for 'alpha'; omit the leading dashes\nMySettingsClass: CLI name 'h' for 'alpha' is reserved by argparse\nMySettingsClass: cli_shortcuts target 'nope' does not match any CLI argument",
+ {"nope": "n", "alpha": ["-a", "h"]},
+ "MySettingsClass: cli_shortcuts target 'nope' does not match any CLI argument\nMySettingsClass: invalid shortcut '-a' for 'alpha'; omit the leading dashes\nMySettingsClass: CLI name 'h' for 'alpha' is reserved by argparse",
id="all_errors_reported",
),
],
diff --git a/tests/cdm_data_loaders/core/test_transforms.py b/tests/cdm_data_loaders/core/test_transforms.py
index 2ee32a61..77828a1d 100644
--- a/tests/cdm_data_loaders/core/test_transforms.py
+++ b/tests/cdm_data_loaders/core/test_transforms.py
@@ -1,10 +1,11 @@
"""Tests for `cdm_data_loaders.core.transforms` and `TransformSettingsMixin`."""
import functools
+import operator
from collections.abc import Iterable
from pathlib import Path
from types import SimpleNamespace
-from typing import Annotated, Any, Final
+from typing import Annotated, Any, Final, Union
import pyarrow as pa
import pytest
@@ -23,14 +24,21 @@
TransformRegistry,
)
from cdm_data_loaders.core.transforms.introspection import (
+ _annotation_capabilities,
+ _annotation_is_transform_context,
+ _safe_type_hints,
+ _unwrap_partial_target,
accepts_pyarrow_table,
accepts_record_dict,
+ inspect_transform_signature,
)
from cdm_data_loaders.core.transforms.routing import (
align_and_cast_table,
compose_transforms,
emit_raw_table,
+ invoke_transform,
make_record_router,
+ merge_routed_outputs,
resolve_schema_map,
route_and_coerce_record,
route_and_coerce_table,
@@ -38,6 +46,7 @@
)
from cdm_data_loaders.core.transforms.settings_mixin import (
LockedTransformHooksMixin,
+ TransformSettingsMixin,
reconcile_multi_table_schema_map,
validate_table_transforms,
)
@@ -860,7 +869,7 @@ def test_emit_raw_table_pass_arrow_and_dict(
def test_emit_raw_table_fail_empty_prefix() -> None:
"""emit_raw_table raises ValueError when prefix is empty."""
ctx = TransformContext.for_source("id_mapping", "dir/sample.dat")
- with pytest.raises(ValueError, match="prefix must be a non-empty string"):
+ with pytest.raises(ValueError, match=r"^prefix must be a non-empty string$"):
emit_raw_table({"id": "1"}, ctx, prefix="")
@@ -1280,35 +1289,78 @@ class LockedExtractSettings(LockedTransformHooksMixin, JsonlExtractSettings, cli
@pytest.mark.parametrize(
- ("use_pyarrow", "transformed_tables", "explicit_schemas", "expected_built", "expected_tables"),
+ (
+ "use_pyarrow",
+ "transformed_tables",
+ "explicit_schemas",
+ "none_default_tables",
+ "expected_built",
+ "expected_registry",
+ ),
[
- pytest.param(False, (), {}, [], None, id="use-pyarrow-false"),
- pytest.param(True, (), {}, ["alpha", "beta"], {"alpha", "beta"}, id="all-defaults"),
- pytest.param(True, ("alpha",), {}, ["beta"], {"beta"}, id="skips-transformed-without-explicit-schema"),
+ pytest.param(False, (), {}, (), [], None, id="use-pyarrow-false"),
+ pytest.param(
+ True,
+ (),
+ {},
+ (),
+ ["alpha", "beta"],
+ {"alpha": ITEMS_SCHEMA, "beta": ITEMS_SCHEMA},
+ id="all-defaults",
+ ),
+ pytest.param(
+ True,
+ ("alpha",),
+ {},
+ (),
+ ["beta"],
+ {"beta": ITEMS_SCHEMA},
+ id="skips-transformed-without-explicit-schema",
+ ),
pytest.param(
True,
("alpha",),
{"alpha_out": SOURCED_ITEMS_SCHEMA},
+ (),
["beta"],
- {"alpha_out", "beta"},
+ {"alpha_out": SOURCED_ITEMS_SCHEMA, "beta": ITEMS_SCHEMA},
id="merges-explicit-fanout-and-untransformed-default",
),
- pytest.param(True, ("alpha", "beta"), {}, [], None, id="all-transformed-unschematized"),
+ pytest.param(
+ True,
+ (),
+ {"alpha": OVERRIDE_SCHEMA},
+ (),
+ ["beta"],
+ {"alpha": OVERRIDE_SCHEMA, "beta": ITEMS_SCHEMA},
+ id="explicit-source-table-override-skips-default-builder",
+ ),
+ pytest.param(
+ True,
+ (),
+ {},
+ ("alpha",),
+ ["alpha", "beta"],
+ {"beta": ITEMS_SCHEMA},
+ id="skips-none-default-schema",
+ ),
+ pytest.param(True, ("alpha", "beta"), {}, (), [], None, id="all-transformed-unschematized"),
],
)
def test_reconcile_multi_table_schema_map_reconciles_defaults_and_overrides(
use_pyarrow: bool,
transformed_tables: tuple[str, ...],
explicit_schemas: dict[str, pa.Schema],
+ none_default_tables: tuple[str, ...],
expected_built: list[str],
- expected_tables: set[str] | None,
+ expected_registry: dict[str, pa.Schema] | None,
) -> None:
"""reconcile_multi_table_schema_map builds defaults only for untransformed unmapped tables and merges explicit schemas."""
built: list[str] = []
def _build_default(table_name: str) -> pa.Schema | None:
built.append(table_name)
- return ITEMS_SCHEMA
+ return None if table_name in none_default_tables else ITEMS_SCHEMA
transforms = (
TransformRegistry(registry=dict.fromkeys(transformed_tables, _identity_items_transform))
@@ -1326,8 +1378,313 @@ def _build_default(table_name: str) -> pa.Schema | None:
)
assert built == expected_built
- if expected_tables is None:
- assert result is None
+ expected = ArrowSchemaRegistry(registry=expected_registry) if expected_registry is not None else None
+ assert result == expected
+
+
+type _CtxAlias = TransformContext
+
+
+def test_unwrap_partial_target_unwraps_nested_partials() -> None:
+ """_unwrap_partial_target unwraps arbitrarily nested functools.partial wrappers."""
+
+ def base_fn(table: pa.Table, *, _flag: bool = False) -> dict[str, Any]:
+ del table, _flag
+ return {}
+
+ nested = functools.partial(functools.partial(base_fn, _flag=True))
+ assert _unwrap_partial_target(nested) is base_fn
+
+
+@pytest.mark.parametrize(
+ "case",
+ [
+ pytest.param("nested_partial", id="nested-partial"),
+ pytest.param("callable_object", id="callable-object"),
+ pytest.param("unresolvable_annotation", id="unresolvable-annotation"),
+ ],
+)
+def test_safe_type_hints_handles_callables_and_errors(case: str) -> None:
+ """_safe_type_hints resolves hints on partials and callable objects and returns empty dict on errors."""
+ if case == "nested_partial":
+
+ def base_fn(table: pa.Table, *, _flag: bool = False) -> dict[str, Any]:
+ del table, _flag
+ return {}
+
+ target: object = functools.partial(functools.partial(base_fn, _flag=True))
+ assert _safe_type_hints(target)["table"] is pa.Table # type: ignore[arg-type]
+ elif case == "callable_object":
+
+ class _CallableObj:
+ def __call__(self, table: pa.Table, ctx: TransformContext | None = None) -> dict[str, Any]:
+ del table, ctx
+ return {}
+
+ assert _safe_type_hints(_CallableObj())["table"] is pa.Table
+ else:
+
+ def bad_hint_fn(_x: object) -> dict[str, Any]:
+ return {}
+
+ bad_hint_fn.__annotations__["_x"] = "MissingLocalType"
+ assert _safe_type_hints(bad_hint_fn) == {}
+
+
+@pytest.mark.parametrize(
+ ("ann", "expected_caps"),
+ [
+ pytest.param(pa.Table, (True, False), id="pa_table"),
+ pytest.param("pa.Table", (True, False), id="str_table"),
+ pytest.param("pa.Table | dict[str, Any]", (True, True), id="str_table_or_dict"),
+ pytest.param("pa.Table | Mapping[str, Any]", (True, True), id="str_table_or_mapping"),
+ pytest.param("dict[str, Any]", (False, True), id="str_dict"),
+ pytest.param(pa.Table | None, (True, False), id="optional_pa_table"),
+ pytest.param(Annotated[pa.Table, "meta"], (True, False), id="annotated_pa_table"),
+ pytest.param(int, (False, True), id="fallback_int"),
+ ],
+)
+def test_annotation_capabilities_classifies_types(ann: object, expected_caps: tuple[bool, bool]) -> None:
+ """_annotation_capabilities classifies table, dict, union, annotated, and string annotations."""
+ assert _annotation_capabilities(ann) == expected_caps
+
+
+@pytest.mark.parametrize(
+ ("ann", "expected"),
+ [
+ pytest.param(TransformContext, True, id="direct"),
+ pytest.param(_CtxAlias, True, id="type_alias"),
+ pytest.param(Annotated[TransformContext, "meta"], True, id="annotated"),
+ pytest.param(Annotated[int, "meta"], False, id="annotated_non_ctx"),
+ pytest.param(Annotated[int, TransformContext], False, id="annotated_ctx_in_metadata_only"),
+ pytest.param(TransformContext | None, True, id="union"),
+ pytest.param(int | None, False, id="union_non_ctx"),
+ pytest.param(operator.getitem(Union, (TransformContext, None)), True, id="typing_union_ctx"),
+ pytest.param(operator.getitem(Union, (int, str)), False, id="typing_union_non_ctx"),
+ pytest.param("TransformContext | None", True, id="str_ctx"),
+ pytest.param("dict[str, Any]", False, id="str_non_ctx"),
+ pytest.param(int, False, id="non_ctx"),
+ ],
+)
+def test_annotation_is_transform_context_detects_variants(ann: object, expected: bool) -> None:
+ """_annotation_is_transform_context detects TransformContext across aliases, unions, Annotated, and strings."""
+ assert _annotation_is_transform_context(ann) is expected
+
+
+@pytest.mark.parametrize(
+ ("case", "expected"),
+ [
+ pytest.param("kwonly_only", (False, True, False), id="kwonly-only"),
+ pytest.param("posonly_arrow", (True, False, True), id="posonly-arrow-and-annotated-ctx"),
+ pytest.param("optional_non_ctx", (False, True, False), id="optional-non-ctx"),
+ pytest.param("required_second_param", (False, True, True), id="required-second-param"),
+ pytest.param("str_annotated", (True, False, True), id="string-annotations"),
+ pytest.param("flagged_attr", (True, True, True), id="accepts-pyarrow-attr"),
+ ],
+)
+def test_inspect_transform_signature_handles_parameter_variants(
+ case: str,
+ expected: tuple[bool, bool, bool],
+) -> None:
+ """inspect_transform_signature classifies kwonly, posonly, optional, string, and flagged callables."""
+ if case == "kwonly_only":
+
+ def fn(*, _flag: bool = False) -> dict[str, Any]:
+ return {}
+
+ elif case == "posonly_arrow":
+
+ def fn(_table: pa.Table, _info: Annotated[TransformContext, "m"] | None = None, /) -> dict[str, Any]: # type: ignore[misc]
+ return {}
+
+ elif case == "optional_non_ctx":
+
+ def fn(_row: dict[str, Any], _limit: int = 10) -> dict[str, Any]: # type: ignore[misc]
+ return {}
+
+ elif case == "required_second_param":
+
+ def fn(_row: dict[str, Any], _extra: int) -> dict[str, Any]: # type: ignore[misc]
+ return {}
+
+ elif case == "str_annotated":
+
+ def fn(_data: "pa.Table", _info: "TransformContext | None" = None) -> dict[str, Any]: # type: ignore[misc]
+ return {}
+
+ else:
+
+ def fn(_x: object) -> dict[str, Any]: # type: ignore[misc]
+ return {}
+
+ fn.accepts_pyarrow = True # type: ignore[attr-defined]
+
+ assert inspect_transform_signature(fn) == expected
+
+
+@pytest.mark.parametrize(
+ "case",
+ [
+ pytest.param("table_to_dict_with_ctx", id="table-to-dict-with-ctx"),
+ pytest.param("dict_to_table_with_input_schema", id="dict-to-table-with-input-schema"),
+ ],
+)
+def test_invoke_transform_adapts_input_type(case: str) -> None:
+ """invoke_transform adapts PyArrow tables for dict transforms and dicts for PyArrow transforms."""
+ ctx = TransformContext.for_source("items", "input.json")
+
+ if case == "table_to_dict_with_ctx":
+
+ def _dict_with_ctx(row: dict[str, Any], info: TransformContext) -> dict[str, list[dict[str, Any]]]:
+ return {"items": [{"id": row["id"], "src": info.source_file}]}
+
+ tbl = pa.table({"id": ["1"]})
+ assert invoke_transform(_dict_with_ctx, tbl, ctx) == {"items": [{"id": "1", "src": "input.json"}]}
+ else:
+ arrow_schema = pa.schema([pa.field("id", pa.string()), pa.field("opt", pa.int64(), nullable=True)])
+
+ def _arrow_only(table: pa.Table) -> dict[str, pa.Table]:
+ return {"items": table}
+
+ coerced_out = invoke_transform(_arrow_only, {"id": "1"}, ctx, input_schema=arrow_schema)
+ assert isinstance(coerced_out["items"], pa.Table)
+ assert coerced_out["items"].to_pylist() == [{"id": "1", "opt": None}]
+
+
+@pytest.mark.parametrize(
+ ("initial", "addition", "expect_arrow", "expected_rows"),
+ [
+ pytest.param({}, {"t": [{"a": 1}]}, False, [{"a": 1}], id="new-table-key"),
+ pytest.param(
+ {"t": pa.table({"a": [1]})},
+ {"t": pa.table({"b": [2]})},
+ True,
+ [{"a": 1, "b": None}, {"a": None, "b": 2}],
+ id="arrow-plus-arrow-mismatched-columns",
+ ),
+ pytest.param(
+ {"t": [{"a": 1}]},
+ {"t": [{"a": 2}]},
+ False,
+ [{"a": 1}, {"a": 2}],
+ id="list-plus-list",
+ ),
+ pytest.param(
+ {"t": pa.table({"a": [1]})},
+ {"t": [{"a": 2}]},
+ False,
+ [{"a": 1}, {"a": 2}],
+ id="arrow-plus-list",
+ ),
+ pytest.param(
+ {"t": [{"a": 1}]},
+ {"t": pa.table({"a": [2]})},
+ False,
+ [{"a": 1}, {"a": 2}],
+ id="list-plus-arrow",
+ ),
+ ],
+)
+def test_merge_routed_outputs_pass_merges_tables_lists_and_mixed(
+ initial: dict[str, list[dict[str, Any]] | pa.Table],
+ addition: dict[str, list[dict[str, Any]] | pa.Table],
+ *,
+ expect_arrow: bool,
+ expected_rows: list[dict[str, Any]],
+) -> None:
+ """merge_routed_outputs merges new keys, mismatched PyArrow tables, row lists, and mixed outputs."""
+ merged: dict[str, list[dict[str, Any]] | pa.Table] = dict(initial)
+ merge_routed_outputs(merged, addition)
+
+ result = merged["t"]
+ if expect_arrow:
+ assert isinstance(result, pa.Table)
+ assert result.to_pylist() == expected_rows
+ else:
+ assert isinstance(result, list)
+ assert result == expected_rows
+
+
+def test_align_and_cast_table_preserves_target_schema_metadata() -> None:
+ """align_and_cast_table preserves metadata attached to the target PyArrow schema."""
+ base_tbl = pa.table({"id": pa.array(["1"], type=pa.string())})
+ target_with_meta = pa.schema([pa.field("id", pa.string())], metadata={b"k": b"v"})
+
+ aligned = align_and_cast_table(base_tbl, target_with_meta)
+
+ assert aligned.schema.metadata == {b"k": b"v"}
+
+
+def test_validate_routed_output_drops_empty_zero_column_table() -> None:
+ """validate_routed_output drops zero-row, zero-column PyArrow tables."""
+ assert validate_routed_output({"empty": pa.table({})}) == {}
+
+
+def test_compose_transforms_fail_empty_list() -> None:
+ """compose_transforms raises ValueError when given an empty list."""
+ with pytest.raises(ValueError, match=r"^transforms must contain at least one callable$"):
+ compose_transforms([])
+
+
+def test_route_and_coerce_table_defaults_to_pyarrow_output() -> None:
+ """route_and_coerce_table returns PyArrow tables by default when use_pyarrow is omitted."""
+ base_tbl = pa.table({"id": pa.array(["1"], type=pa.string())})
+
+ default_routed = route_and_coerce_table(base_tbl, "items")
+
+ assert isinstance(default_routed["items"], pa.Table)
+ assert default_routed["items"].to_pylist() == [{"id": "1"}]
+
+
+@pytest.mark.parametrize(
+ ("use_multi_table", "loader_file_format", "expected_keys"),
+ [
+ pytest.param(False, "parquet", {"custom"}, id="no-table-name-parquet"),
+ pytest.param(False, "jsonl", None, id="no-table-name-jsonl"),
+ pytest.param(True, "parquet", {"custom", "other"}, id="multi-table-parquet"),
+ pytest.param(True, "jsonl", None, id="multi-table-jsonl"),
+ ],
+)
+def test_transform_settings_mixin_multi_table_and_no_table_name(
+ monkeypatch: pytest.MonkeyPatch,
+ use_multi_table: bool,
+ loader_file_format: str,
+ expected_keys: set[str] | None,
+) -> None:
+ """TransformSettingsMixin handles multi-table default schema reconciliation and subclasses without table_name."""
+ items_schema = ITEMS_SCHEMA
+ other_schema = pa.schema([pa.field("other_id", pa.string())])
+ mod_name = register_temp_module(
+ monkeypatch,
+ "_test_settings_mixin_multi_and_none",
+ TABLE_TRANSFORMS={"items": _identity_items_transform},
+ OUTPUT_SCHEMAS={"custom": items_schema},
+ )
+
+ class _NoTableNameSettings(TransformSettingsMixin, BaseModel):
+ loader_file_format: str = "parquet"
+
+ class _MultiTableSettings(_NoTableNameSettings):
+ def _resolve_settings_schema_map(self, default_pyarrow_schema: pa.Schema | None = None) -> None:
+ del default_pyarrow_schema
+ self._resolve_multi_table_schema_map({"items": items_schema, "other": other_schema})
+
+ if use_multi_table:
+ settings: TransformSettingsMixin = _MultiTableSettings(
+ loader_file_format=loader_file_format,
+ transforms_module=mod_name,
+ output_schemas_module=mod_name,
+ )
+ else:
+ settings = _NoTableNameSettings(
+ loader_file_format=loader_file_format,
+ output_schemas_module=mod_name,
+ )
+
+ if expected_keys is None:
+ assert settings.schema_map is None
else:
- assert result is not None
- assert set(result.registry) == expected_tables
+ expected_registry = {"custom": items_schema}
+ if "other" in expected_keys:
+ expected_registry["other"] = other_schema
+ assert settings.schema_map == ArrowSchemaRegistry(registry=expected_registry)
diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py
index 17d2da9c..9acc9b75 100644
--- a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py
+++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py
@@ -1,9 +1,10 @@
"""Tests for ncbi_ftp.assembly module — path helpers, file filtering, and FTP assembly downloads."""
from collections.abc import Callable
+from contextlib import nullcontext
from ftplib import FTP
from pathlib import Path, PurePosixPath
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
import pytest
@@ -251,3 +252,59 @@ def fake_retrbinary(_cmd: str, callback: Callable[[bytes], None]) -> None:
assert result["files_skipped_checksum_mismatch"] == 1
assert not (dest_dir / "GCF_000001215.4_genomic.fna.gz").exists()
assert not (dest_dir / "GCF_000001215.4_genomic.fna.gz.md5").exists()
+
+
+def test_download_assembly_to_local_no_md5_manifest_and_keepalive_args(
+ tmp_path: Path,
+) -> None:
+ """download_assembly_to_local forwards ftp_host, retries=1, checksum_fn, and keepalive timestamps."""
+ mock_ftp = MagicMock(spec=FTP)
+ assembly_ftp_path = PurePosixPath("/") / "genomes" / "all" / ACC_PATH_215
+ keepalive_calls: list[float] = []
+ download_calls: list[tuple[PurePosixPath, int, str | None]] = []
+
+ def fake_keepalive(_ftp: FTP, last_ts: float) -> float:
+ keepalive_calls.append(last_ts)
+ return last_ts + 10.0
+
+ def fake_download(
+ _ftp: FTP,
+ remote: PurePosixPath,
+ local: Path,
+ *,
+ retries: int = 3,
+ checksum_fn: str | None = None,
+ ) -> str | None:
+ download_calls.append((remote, retries, checksum_fn))
+ local.write_bytes(b"data")
+ return None
+
+ with (
+ patch("cdm_data_loaders.datasets.ncbi_ftp.assembly.ftp_session", return_value=nullcontext(mock_ftp)) as m_sess,
+ patch(
+ "cdm_data_loaders.datasets.ncbi_ftp.assembly.ftp_list_dir",
+ return_value=[
+ PurePosixPath("GCF_000001215.4_genomic.fna.gz"),
+ PurePosixPath("GCF_000001215.4_assembly_stats.txt"),
+ ],
+ ),
+ patch("cdm_data_loaders.datasets.ncbi_ftp.assembly.ftp_noop_keepalive", side_effect=fake_keepalive),
+ patch("cdm_data_loaders.datasets.ncbi_ftp.assembly.ftp_download_file", side_effect=fake_download),
+ patch("cdm_data_loaders.datasets.ncbi_ftp.assembly.time.monotonic", side_effect=[100.0, 200.0, 300.0]),
+ ):
+ stats = download_assembly_to_local(assembly_ftp_path, tmp_path, ftp_host="ftp.custom.org")
+
+ m_sess.assert_called_once_with("ftp.custom.org", None)
+ assert stats == {
+ "accession": "GCF_000001215.4",
+ "assembly_dir": "GCF_000001215.4_Release_6_plus_ISO1_MT",
+ "files_downloaded": 2,
+ "files_skipped_checksum_mismatch": 0,
+ "files_without_checksum": 2,
+ }
+ assert keepalive_calls == [100.0, 200.0]
+ assert download_calls == [
+ (PurePosixPath("GCF_000001215.4_genomic.fna.gz"), 1, None),
+ (PurePosixPath("GCF_000001215.4_assembly_stats.txt"), 1, None),
+ ]
+ assert not (tmp_path / "raw_data" / ACC_PATH_215 / "md5checksums.txt").exists()
diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py
index babffba2..c1f6bfde 100644
--- a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py
+++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py
@@ -746,3 +746,104 @@ def test_download_assembly_summary_unknown_database() -> None:
"""Raise ValueError when database is neither refseq nor genbank."""
with pytest.raises(ValueError, match="Unknown database: bogus"):
download_assembly_summary(database="bogus")
+
+
+@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.list_objects")
+def test_scan_store_to_synthetic_summary_invokes_progress_callback(
+ mock_list_objects: MagicMock,
+) -> None:
+ """scan_store_to_synthetic_summary invokes progress_callback with (count, accession)."""
+ mock_list_objects.return_value = [
+ {"Key": "prefix/GCF_000001215.4_Release_6/file.gz", "LastModified": datetime(2024, 1, 1, tzinfo=UTC)},
+ ]
+ progress_calls: list[tuple[int, str]] = []
+
+ def _record_progress(count: int, acc: str) -> None:
+ progress_calls.append((count, acc))
+
+ scan_store_to_synthetic_summary(
+ PurePosixPath("test-bucket"),
+ PurePosixPath("prefix"),
+ "2024/01/31",
+ progress_callback=_record_progress,
+ )
+
+ assert progress_calls == [(1, "GCF_000001215.4")]
+
+
+@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.list_objects")
+def test_scan_store_to_synthetic_summary_logs_and_reraises_error(
+ mock_list_objects: MagicMock,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """scan_store_to_synthetic_summary logs an error and reraises when listing objects fails."""
+ mock_list_objects.side_effect = RuntimeError("s3 down")
+
+ with (
+ caplog.at_level("ERROR", logger="cdm_data_loaders.datasets.ncbi_ftp.manifest"),
+ pytest.raises(RuntimeError, match="s3 down"),
+ ):
+ scan_store_to_synthetic_summary(PurePosixPath("test-bucket"), PurePosixPath("prefix"), "2024/01/31")
+
+ assert "Error scanning store" in caplog.text
+
+
+@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.get_existing_object_info")
+@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.ftp_retrieve_text")
+@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.ftp_noop_keepalive", side_effect=lambda _f, t: t)
+@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.connect_ftp")
+@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.list_objects")
+def test_verify_transfer_candidates_custom_host_progress_and_debug_logs(
+ mock_list_objects: MagicMock,
+ mock_connect_ftp: MagicMock,
+ mock_keepalive: MagicMock,
+ mock_retrieve: MagicMock,
+ mock_obj_info: MagicMock,
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """verify_transfer_candidates forwards custom ftp_host, invokes progress_callback, and logs debug and info."""
+ custom_host = "ftp.mirror.example.org"
+ rec_match = AssemblyRecord(
+ accession="GCF_000001215.4",
+ status="latest",
+ seq_rel_date="2024/01/01",
+ ftp_url=f"https://{custom_host}/genomes/all/GCF/000/001/215/GCF_000001215.4_R6",
+ assembly_dir=PurePosixPath("GCF_000001215.4_R6"),
+ )
+ rec_mismatch = AssemblyRecord(
+ accession="GCF_000001405.40",
+ status="latest",
+ seq_rel_date="2024/01/01",
+ ftp_url=f"https://{custom_host}/genomes/all/GCF/000/001/405/GCF_000001405.40_GRCh38",
+ assembly_dir=PurePosixPath("GCF_000001405.40_GRCh38"),
+ )
+ mock_list_objects.return_value = [{"Key": "k"}]
+ mock_connect_ftp.return_value = MagicMock()
+ digest = "a" * 32
+ mock_retrieve.side_effect = [
+ f"{digest} ./GCF_000001215.4_R6_genomic.fna.gz\n",
+ f"{digest} ./GCF_000001405.40_GRCh38_genomic.fna.gz\n",
+ ]
+ mock_obj_info.side_effect = [{"Metadata": {"md5": digest}}, None]
+
+ progress: list[tuple[int, int, str]] = []
+
+ def _record_progress(done: int, total: int, acc: str) -> None:
+ progress.append((done, total, acc))
+
+ with caplog.at_level("DEBUG", logger="cdm_data_loaders.datasets.ncbi_ftp.manifest"):
+ confirmed = verify_transfer_candidates(
+ ["GCF_000001215.4", "GCF_000001405.40"],
+ {"GCF_000001215.4": rec_match, "GCF_000001405.40": rec_mismatch},
+ PurePosixPath("test-bucket"),
+ PurePosixPath("prefix"),
+ ftp_host=custom_host,
+ progress_callback=_record_progress,
+ )
+
+ assert confirmed == ["GCF_000001405.40"]
+ assert mock_keepalive.called
+ assert progress == [(1, 2, "GCF_000001215.4"), (2, 2, "GCF_000001405.40")]
+ assert "Pruned GCF_000001215.4" in caplog.text
+ assert "File missing from store:" in caplog.text
+ assert "Checksum verification: 1 confirmed (0 missing from store), 1 pruned (of 2 candidates)" in caplog.text
diff --git a/tests/cdm_data_loaders/io/test_clients.py b/tests/cdm_data_loaders/io/test_clients.py
index a7039cb0..1648ed0c 100644
--- a/tests/cdm_data_loaders/io/test_clients.py
+++ b/tests/cdm_data_loaders/io/test_clients.py
@@ -3,6 +3,7 @@
import hashlib
import logging
from pathlib import Path
+from typing import BinaryIO, Self
from unittest.mock import AsyncMock, MagicMock, patch
import httpx
@@ -38,7 +39,8 @@ async def test_download_to_file_system_success(
) -> None:
"""Simple happy path test of a successful download."""
- def handler(_: httpx.Request) -> httpx.Response:
+ def handler(request: httpx.Request) -> httpx.Response:
+ assert request.method == "GET"
return httpx.Response(
200,
content=sample_content,
@@ -59,20 +61,48 @@ def handler(_: httpx.Request) -> httpx.Response:
@pytest.mark.asyncio
-async def test_chunked_streaming(tmp_path: Path, downloader_adapter: DownloaderAdapter) -> None:
- """Ensure that chunked downloads work correctly."""
+async def test_chunked_streaming(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, downloader_adapter: DownloaderAdapter
+) -> None:
+ """Ensure that chunked downloads respect the configured chunk size."""
content = b"a" * 10_000
+ chunk_size = 1024
+ written_sizes: list[int] = []
+ real_open = Path.open
- def handler(_: httpx.Request) -> httpx.Response:
- return httpx.Response(200, content=content)
+ class TrackingWriter:
+ def __init__(self, fh: BinaryIO) -> None:
+ self._fh = fh
+
+ def write(self, data: bytes) -> int:
+ written_sizes.append(len(data))
+ return self._fh.write(data)
- downloader = downloader_adapter.make_downloader(handler, chunk_size=1024)
+ def __enter__(self) -> Self:
+ self._fh.__enter__()
+ return self
+ def __exit__(self, *args: object) -> None:
+ self._fh.__exit__(*args)
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ assert request.method == "GET"
+ return httpx.Response(200, content=content)
+
+ downloader = downloader_adapter.make_downloader(handler, chunk_size=chunk_size)
destination = tmp_path / "file.txt"
+
+ def tracking_open(self: Path, mode: str = "r", *args: object, **kwargs: object) -> object:
+ fh = real_open(self, mode, *args, **kwargs) # type: ignore[call-overload]
+ if self == destination and mode == "wb":
+ return TrackingWriter(fh)
+ return fh
+
+ monkeypatch.setattr(Path, "open", tracking_open)
await downloader_adapter.download(downloader, DOWNLOAD_URL, destination)
- # TODO: add in count of number of times fh.write is called by _download_once?
assert destination.read_bytes() == content
+ assert written_sizes == [chunk_size] * 9 + [len(content) - chunk_size * 9]
@pytest.mark.parametrize(
@@ -212,7 +242,7 @@ async def test_checksum_invalid_downloader(
@pytest.mark.asyncio
async def test_checksum_mismatch(tmp_path: Path, sample_content: bytes, downloader_adapter: DownloaderAdapter) -> None:
- """Test checksum checking!"""
+ """Test checksum mismatch raises ChecksumMismatchError."""
def handler(_: httpx.Request) -> httpx.Response:
return httpx.Response(200, content=sample_content)
diff --git a/tests/cdm_data_loaders/parsers/test_checkm2.py b/tests/cdm_data_loaders/parsers/test_checkm2.py
index 10196f9c..d6d0d62f 100644
--- a/tests/cdm_data_loaders/parsers/test_checkm2.py
+++ b/tests/cdm_data_loaders/parsers/test_checkm2.py
@@ -63,7 +63,7 @@ def test_get_checkm2_data_headers_only(test_data_dir: Path) -> None:
with pytest.raises(
RuntimeError,
- match=re.escape("no valid data found in checkm2_file"),
+ match=f"^{re.escape('no valid data found in checkm2_file')}$",
):
get_checkm2_data(full_path)
@@ -83,3 +83,18 @@ def test_get_checkm2_scores(key: str, test_data_dir: Path) -> None:
"""Test that checkm2 scores are correctly parsed."""
results_dir = test_data_dir / f"results_{key}" / "quality_report.tsv"
assert get_checkm2_data(results_dir) == RESULTS[key]["checkm2"]
+
+
+def test_get_checkm2_data_empty_metric_values(tmp_path: Path) -> None:
+ """Return None for empty completeness or contamination values."""
+ tsv_path = tmp_path / "quality_report.tsv"
+ tsv_path.write_text(
+ "Name\tCompleteness\tContamination\nbin_empty_contam\t95.5\t\nbin_empty_comp\t\t1.25\nbin_both_empty\t\t\n",
+ encoding="utf-8",
+ )
+
+ assert get_checkm2_data(tsv_path) == {
+ "bin_empty_contam": {"checkm2_completeness": 95.5, "checkm2_contamination": None},
+ "bin_empty_comp": {"checkm2_completeness": None, "checkm2_contamination": 1.25},
+ "bin_both_empty": {"checkm2_completeness": None, "checkm2_contamination": None},
+ }
diff --git a/tests/cdm_data_loaders/parsers/test_uniprot_metalink.py b/tests/cdm_data_loaders/parsers/test_uniprot_metalink.py
index 083330b8..65815e4c 100644
--- a/tests/cdm_data_loaders/parsers/test_uniprot_metalink.py
+++ b/tests/cdm_data_loaders/parsers/test_uniprot_metalink.py
@@ -1,11 +1,13 @@
"""Tests for UniProt metalink file parser."""
import logging
+import re
from pathlib import Path
+from xml.etree.ElementTree import ElementTree
import pytest
-from cdm_data_loaders.parsers.uniprot_metalink import generate_data_source_table, get_files
+from cdm_data_loaders.parsers.uniprot_metalink import generate_data_source_table, get_files, parse_metalink
def test_generate_data_source_table(test_data_dir: Path) -> None:
@@ -31,17 +33,40 @@ def test_generate_data_source_table(test_data_dir: Path) -> None:
("missing", ["license", "publisher"]),
],
)
-def test_generate_data_source_error_missing_fields(test_data_dir: Path, file: str, missing: list[str]) -> None:
+def test_generate_data_source_error_missing_fields(
+ test_data_dir: Path, file: str, missing: list[str], caplog: pytest.LogCaptureFixture
+) -> None:
"""Test the parsing of the data source table."""
metalink_path = test_data_dir / "uniprot" / "metalink" / "invalid" / f"{file}_metalink.xml"
missing_fields = ", ".join(missing)
+ expected_msg = f"Missing required elements from metalink file: {missing_fields}"
with pytest.raises(
RuntimeError,
- match=f"Missing required elements from metalink file: {missing_fields}",
+ match=f"^{re.escape(expected_msg)}$",
):
generate_data_source_table(str(metalink_path))
+ assert len(caplog.records) == 1
+ assert caplog.records[0].levelno == logging.ERROR
+ assert caplog.records[0].getMessage() == expected_msg
+
+
+def test_parse_metalink_missing_root(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture
+) -> None:
+ """Log and raise RuntimeError when the parsed XML document has no root element."""
+ metalink_path = tmp_path / "empty.xml"
+ monkeypatch.setattr("cdm_data_loaders.parsers.uniprot_metalink.parse", lambda _path: ElementTree())
+
+ expected_msg = f"Could not find root in metalink file: {metalink_path}"
+ with pytest.raises(RuntimeError, match=f"^{re.escape(expected_msg)}$"):
+ parse_metalink(metalink_path)
+
+ assert len(caplog.records) == 1
+ assert caplog.records[0].levelno == logging.ERROR
+ assert caplog.records[0].getMessage() == expected_msg
+
UNIPROT_METALINK_DATA = {
"uniprot_trembl.fasta.gz": {
@@ -152,3 +177,32 @@ def test_get_files(
assert caplog.records[0].getMessage() == msg
else:
assert not caplog.records
+
+
+def test_get_files_without_verification(tmp_path: Path) -> None:
+ """Return None for checksum fields when a file entry has no verification hash."""
+ metalink_path = tmp_path / "no_hash.xml"
+ metalink_path.write_text(
+ '\n'
+ '\n'
+ " \n"
+ ' \n'
+ " 42\n"
+ " \n"
+ ' https://example.org/no_hash.txt\n'
+ " \n"
+ " \n"
+ " \n"
+ "\n",
+ encoding="utf-8",
+ )
+
+ assert get_files(metalink_path) == {
+ "no_hash.txt": {
+ "name": "no_hash.txt",
+ "size": "42",
+ "checksum": None,
+ "checksum_fn": None,
+ "url": "https://example.org/no_hash.txt",
+ }
+ }
diff --git a/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py b/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py
index df566b76..68bb3582 100644
--- a/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py
+++ b/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py
@@ -627,3 +627,31 @@ def record_download_thread(**_kwargs: Any) -> None: # noqa: ANN401
assert mock_download_client.download.call_count == len(batches)
assert main_tid not in download_threads
assert len(download_threads) > 1
+
+
+def test_osf_file_downloader_pass_tags_downloaded_files_table(
+ test_settings: AtbSettings,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """osf_file_downloader tags yielded items with the downloaded_files table name."""
+ monkeypatch.setattr(all_the_bacteria, "download_osf_files", lambda _dir, _files: [{"file_id": "f1"}])
+ items = list(osf_file_downloader(test_settings, []))
+ assert len(items) == 1
+ assert items[0].meta.table_name == "downloaded_files"
+
+
+def test_run_atb_pipeline_pass_passes_dataset_name(
+ test_settings: AtbSettings,
+) -> None:
+ """run_atb_pipeline forwards test settings and DATASET_NAME to build_file_ingest_kwargs."""
+ with (
+ patch.object(
+ all_the_bacteria,
+ "build_file_ingest_kwargs",
+ wraps=all_the_bacteria.build_file_ingest_kwargs,
+ ) as mock_kwargs,
+ patch.object(all_the_bacteria, "run_pipeline", return_value=None),
+ ):
+ run_atb_pipeline(test_settings)
+
+ assert mock_kwargs.call_args.args == (test_settings, DATASET_NAME)
diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py
index 340d917b..82783008 100644
--- a/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py
+++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py
@@ -312,3 +312,48 @@ def fail_transform(_row: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
],
"matched_files": ["alpha/part.jsonl", "zeta/part.jsonl"],
}
+
+
+def test_build_entity_resource_wires_raw_and_validated_options(
+ settings_factory: Callable[..., JsonlPydanticIngestSettings],
+) -> None:
+ """build_entity_resource forwards raw resource name, max_table_nesting=0, and parallelized=True."""
+ settings = settings_factory()
+ with (
+ patch.object(
+ jsonlines_base_module,
+ "build_filesystem_resource",
+ wraps=jsonlines_base_module.build_filesystem_resource,
+ ) as mock_fs_resource,
+ patch.object(
+ jsonlines_base_module.dlt,
+ "transformer",
+ wraps=dlt.transformer,
+ ) as mock_transformer,
+ ):
+ build_entity_resource("widget", settings, _resolve_negative_count_error)
+
+ assert mock_fs_resource.call_args.kwargs["name"] == "widget_raw"
+ assert mock_fs_resource.call_args.kwargs["max_table_nesting"] == 0
+ assert mock_transformer.call_args.kwargs["parallelized"] is True
+
+
+def test_run_registry_ingest_falls_back_when_dataset_name_none(
+ settings_factory: Callable[..., JsonlPydanticIngestSettings],
+ widget_model: type[BaseModel],
+) -> None:
+ """run_registry_ingest falls back to default dataset_name when settings.dataset_name is None."""
+ settings = settings_factory().model_copy(update={"dataset_name": None})
+ mock_load_info = MagicMock(spec=LoadInfo)
+ with patch.object(jsonlines_base_module, "run_pipeline", return_value=mock_load_info) as mock_run_pipeline:
+ result = run_registry_ingest(
+ settings=settings,
+ registry={"widget": widget_model},
+ build_resource=lambda table_name, _entry, cfg: build_entity_resource(
+ table_name, cfg, _resolve_negative_count_error
+ ),
+ pipeline_name="fallback_pipe",
+ )
+
+ assert result is mock_load_info
+ assert mock_run_pipeline.call_args.kwargs["pipeline_kwargs"]["dataset_name"] == "jsonlines_pydantic"
diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py
index 3b5a4b4f..6beb6d05 100644
--- a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py
+++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py
@@ -288,16 +288,27 @@ def fail_transform(_row: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
}
-def test_run_jsonlines_pipeline_uses_subclass_cli_prog_name(
+@pytest.mark.parametrize(
+ ("cli_prog_name", "expected_pipeline_name"),
+ [
+ pytest.param("preset_extract", "preset_extract", id="subclass-cli-prog-name"),
+ pytest.param(None, EXTRACT_PIPELINE_NAME, id="unset-cli-prog-name-fallback"),
+ ],
+)
+def test_run_jsonlines_pipeline_resolves_pipeline_name(
extract_settings_factory: Callable[..., JsonlExtractSettings],
+ cli_prog_name: str | None,
+ expected_pipeline_name: str,
) -> None:
- """run_jsonlines_pipeline forwards a preset subclass cli_prog_name as pipeline_name."""
+ """run_jsonlines_pipeline uses subclass cli_prog_name or falls back to EXTRACT_PIPELINE_NAME."""
- class PresetExtractSettings(JsonlExtractSettings, cli_prog_name="preset_extract"):
+ class CustomExtractSettings(JsonlExtractSettings, cli_prog_name=cli_prog_name):
pass
base_settings = extract_settings_factory()
- preset_settings = PresetExtractSettings.model_validate(
+ custom_settings = CustomExtractSettings.model_validate(
base_settings.model_dump(exclude=set(JsonlExtractSettings.model_computed_fields))
)
- assert_core_run_pipeline_args(extract_pipeline_module, run_jsonlines_pipeline, preset_settings, "preset_extract")
+ assert_core_run_pipeline_args(
+ extract_pipeline_module, run_jsonlines_pipeline, custom_settings, expected_pipeline_name
+ )
diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py
index dc0c94de..77ee3cbe 100644
--- a/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py
+++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py
@@ -1,5 +1,6 @@
"""Integration tests for build_entity_resource: file reading, validation, and dlt wiring together."""
+import json
from collections.abc import Callable
from pathlib import Path
from typing import Any
@@ -7,12 +8,14 @@
import dlt
import pyarrow as pa
import pytest
-from pydantic import BaseModel
+from pydantic import BaseModel, Field, field_validator
from cdm_data_loaders.converters.facades.pydantic_to_pyarrow import PydanticToPyArrow
from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA
from cdm_data_loaders.core.preprocessors import FilePreprocessError
from cdm_data_loaders.pipelines.jsonlines.pydantic import (
+ PYDANTIC_PIPELINE_NAME,
+ _make_validator_callbacks,
build_entity_resource,
run_jsonlines_with_validation_pipeline,
)
@@ -426,20 +429,87 @@ def reject_corrupt_file(file_item: Any) -> Any: # noqa: ANN401
assert row["error_detail"] == "corrupt file header"
-def test_run_jsonlines_with_validation_pipeline_uses_subclass_cli_prog_name(
+@pytest.mark.parametrize(
+ ("cli_prog_name", "expected_pipeline_name"),
+ [
+ pytest.param("preset_pydantic", "preset_pydantic", id="subclass-cli-prog-name"),
+ pytest.param(None, PYDANTIC_PIPELINE_NAME, id="default-pipeline-name"),
+ ],
+)
+def test_run_jsonlines_with_validation_pipeline_resolves_pipeline_name(
scenario_input_dir: Callable[[str], str],
settings_factory: Callable[..., JsonlPydanticIngestSettings],
tmp_path: Path,
+ cli_prog_name: str | None,
+ expected_pipeline_name: str,
) -> None:
- """run_jsonlines_with_validation_pipeline uses a preset subclass cli_prog_name as pipeline_name."""
+ """run_jsonlines_with_validation_pipeline uses subclass cli_prog_name or defaults to PYDANTIC_PIPELINE_NAME."""
+ settings = settings_factory(input_dir=scenario_input_dir("happy_path"), output_dir=str(tmp_path / "out"))
+ if cli_prog_name is not None:
- class PresetPydanticSettings(JsonlPydanticIngestSettings, cli_prog_name="preset_pydantic"):
- pass
+ class PresetPydanticSettings(JsonlPydanticIngestSettings, cli_prog_name=cli_prog_name):
+ pass
- base_settings = settings_factory(input_dir=scenario_input_dir("happy_path"), output_dir=str(tmp_path / "out"))
- preset_settings = PresetPydanticSettings.model_validate(
- base_settings.model_dump(exclude=set(JsonlPydanticIngestSettings.model_computed_fields))
- )
- load_info = assert_run_ok(run_jsonlines_with_validation_pipeline(preset_settings))
+ settings = PresetPydanticSettings.model_validate(
+ settings.model_dump(exclude=set(JsonlPydanticIngestSettings.model_computed_fields))
+ )
+
+ load_info = assert_run_ok(run_jsonlines_with_validation_pipeline(settings))
+
+ assert load_info.pipeline.pipeline_name == expected_pipeline_name
+
+
+class AliasWidget(BaseModel):
+ """Test model with aliased field and custom count validator."""
+
+ widget_id: str = Field(alias="widgetId")
+ count: int
+
+ @field_validator("count")
+ @classmethod
+ def _must_be_positive(cls, value: int) -> int:
+ if value < 0:
+ msg = "count must be non-negative"
+ raise ValueError(msg)
+ return value
- assert load_info.pipeline.pipeline_name == "preset_pydantic"
+
+def test_make_validator_callbacks_caches_valid_model_and_serializes_errors() -> None:
+ """_make_validator_callbacks caches validated models, handles uncached dicts, and serializes non-JSON error contexts."""
+ resolve_error, extract = _make_validator_callbacks(AliasWidget)
+ record: dict[str, Any] = {"widgetId": "w1", "count": 2}
+ assert resolve_error(record) is None
+ extracted_model = extract(record)
+ assert isinstance(extracted_model, AliasWidget)
+ assert extracted_model.widget_id == "w1"
+
+ # different dict identity triggers fresh validation without relying on stale cache
+ extracted_uncached = extract({"widgetId": "w2", "count": 3})
+ assert isinstance(extracted_uncached, AliasWidget)
+ assert extracted_uncached.widget_id == "w2"
+
+ # validation error with ValueError in ctx serializes via default=str and clears cached state
+ assert resolve_error(record) is None
+ error_json = resolve_error({"widgetId": "w_bad", "count": -1})
+ assert error_json is not None
+ parsed_errors = json.loads(error_json)
+ assert "count must be non-negative" in parsed_errors[0]["msg"]
+
+
+@pytest.mark.parametrize(
+ ("extra_kwargs", "record", "expected"),
+ [
+ pytest.param({}, {"widgetId": "w3", "count": 4}, {"widget_id": "w3", "count": 4}, id="default-by-alias-false"),
+ pytest.param(
+ {"by_alias": True}, {"widgetId": "w4", "count": 5}, {"widgetId": "w4", "count": 5}, id="by-alias-true"
+ ),
+ ],
+)
+def test_make_validator_callbacks_dump_to_dict_respects_by_alias(
+ extra_kwargs: dict[str, Any],
+ record: dict[str, Any],
+ expected: dict[str, Any],
+) -> None:
+ """_make_validator_callbacks with dump_to_dict=True uses Python field names by default and aliases when by_alias=True."""
+ _, extract = _make_validator_callbacks(AliasWidget, dump_to_dict=True, **extra_kwargs)
+ assert extract(record) == expected
diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_routing.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_routing.py
index eba9199c..9ed94cf4 100644
--- a/tests/cdm_data_loaders/pipelines/jsonlines/test_routing.py
+++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_routing.py
@@ -410,6 +410,51 @@ def test_make_page_router_pass_buffer_size_controls_batching(
{"_rejected": [PREPROCESS_REJECTED_ROW]},
id="preprocessor-rejected-dict",
),
+ pytest.param(
+ [
+ PREPROCESS_REJECTED_ROW,
+ make_item(
+ record={"widget_id": "ok", "count": 1}, raw_record='{"widget_id": "ok", "count": 1}', line_no=1
+ ),
+ ],
+ {
+ "widget": [{"widget_id": "ok", "count": 1}],
+ "_rejected": [PREPROCESS_REJECTED_ROW],
+ },
+ id="pre-rejected-followed-by-valid",
+ ),
+ pytest.param(
+ [
+ {
+ **make_item(
+ record={"widget_id": "w2", "count": 2}, raw_record='{"widget_id": "w2", "count": 2}', line_no=2
+ ),
+ "error_detail": "ignored",
+ },
+ ],
+ {"widget": [{"widget_id": "w2", "count": 2}]},
+ id="valid-with-extra-error-detail-key",
+ ),
+ pytest.param(
+ [
+ {
+ **make_item(record=None, raw_record="{bad", parse_error="boom", line_no=3),
+ "error_detail": "unrelated",
+ },
+ ],
+ {
+ "_rejected": [
+ {
+ "table_name": "widget",
+ "source_file": "file.jsonl",
+ "line_no": 3,
+ "raw_record": "{bad",
+ "error_detail": "boom",
+ }
+ ],
+ },
+ id="parse-error-precedes-error-detail-key",
+ ),
],
)
def test_make_page_router_pass_use_pyarrow_false_yields_lists_of_dicts(
@@ -426,3 +471,24 @@ def test_make_page_router_pass_use_pyarrow_false_yields_lists_of_dicts(
assert len(results) == len(expected_pages)
assert all(isinstance(page_data, list) for page_data in by_table.values())
assert by_table == expected_pages
+
+
+def test_route_jsonl_lines_coerces_types_via_schema_map(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """route_jsonl_lines passes settings.schema_map to make_record_router so scalar types are coerced before Arrow buffering."""
+ directory = tmp_path / "widget"
+ directory.mkdir()
+ (directory / "data.jsonl").write_text('{"widget_id": 123}\n', encoding="utf-8")
+ mod_name = register_temp_module(
+ monkeypatch,
+ "_test_route_jsonl_lines_coercion",
+ OUTPUT_SCHEMAS={"widget": pa.schema([pa.field("widget_id", pa.string())])},
+ )
+ settings = make_extract_settings(tmp_path, output_schemas_module=mod_name)
+
+ pages = list(route_jsonl_lines(iter(read_items(directory)), settings))
+ assert len(pages) == 1
+ assert isinstance(pages[0].data, pa.Table)
+ assert pages[0].data.to_pylist() == [{"widget_id": "123"}]
diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_settings.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_settings.py
index 811852ea..e9cae5d8 100644
--- a/tests/cdm_data_loaders/pipelines/jsonlines/test_settings.py
+++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_settings.py
@@ -29,6 +29,7 @@
TransformRegistry,
)
from cdm_data_loaders.core.settings import CdmDataLoadersBase
+from cdm_data_loaders.core.transforms.settings_mixin import TransformSettingsMixin
from cdm_data_loaders.datasets.ncbi_datasets.hooks import (
ANNOTATION_REPORT_TABLE,
DATASET_REPORT_TABLE,
@@ -991,7 +992,17 @@ def test_jsonl_pydantic_settings_fail_invalid_max_recursion_depth(
)
-def test_build_output_schema_registry_pass_forwards_max_recursion_depth_and_by_alias() -> None:
+@pytest.mark.parametrize(
+ ("converter_kwargs", "expected_names"),
+ [
+ pytest.param({"max_recursion_depth": 1, "by_alias": True}, ["nodeId", "child"], id="custom-depth-and-alias"),
+ pytest.param({}, ["node_id", "child"], id="default-options"),
+ ],
+)
+def test_build_output_schema_registry_pass_forwards_max_recursion_depth_and_by_alias(
+ converter_kwargs: dict[str, Any],
+ expected_names: list[str],
+) -> None:
"""build_output_schema_registry forwards max_recursion_depth and by_alias to PydanticToPyArrow."""
class RecursiveNode(BaseModel):
@@ -1001,7 +1012,32 @@ class RecursiveNode(BaseModel):
RecursiveNode.model_rebuild()
registry = ModelRegistry(registry={"node": RecursiveNode})
- schema_map = build_output_schema_registry(registry, max_recursion_depth=1, by_alias=True)
+ schema_map = build_output_schema_registry(registry, **converter_kwargs)
assert schema_map is not None
- expected = PydanticToPyArrow(max_recursion_depth=1, by_alias=True).convert(RecursiveNode)
+ expected = PydanticToPyArrow(**converter_kwargs).convert(RecursiveNode)
assert schema_map.registry["node"].equals(expected, check_metadata=True)
+ assert schema_map.registry["node"].names == expected_names
+
+
+def test_jsonl_ingest_settings_resolve_entity_registry_schema_map_defaults(
+ test_defaults: dict[str, Any],
+) -> None:
+ """JsonlIngestSettings._resolve_entity_registry_schema_map defaults max_recursion_depth and by_alias=False."""
+
+ class PlainTransformSettings(JsonlIngestSettings, TransformSettingsMixin):
+ def resolve_entity_schema_map(self, registry: ModelRegistry) -> None:
+ self._resolve_entity_registry_schema_map(registry)
+
+ class AliasedItem(BaseModel):
+ item_id: str = Field(serialization_alias="itemId")
+
+ (test_defaults["input_dir"] / "node").mkdir(exist_ok=True)
+ settings = PlainTransformSettings(
+ **test_defaults,
+ dataset_name="test_ds",
+ loader_file_format="parquet",
+ )
+ registry = ModelRegistry(registry={"node": AliasedItem})
+ settings.resolve_entity_schema_map(registry)
+ assert settings.schema_map is not None
+ assert settings.schema_map.registry["node"].names == ["item_id"]
diff --git a/tests/cdm_data_loaders/pipelines/test_runner.py b/tests/cdm_data_loaders/pipelines/test_runner.py
index c93c6a2f..a4e0178b 100644
--- a/tests/cdm_data_loaders/pipelines/test_runner.py
+++ b/tests/cdm_data_loaders/pipelines/test_runner.py
@@ -139,17 +139,23 @@ def test_settings_missing_sync_attrs_pass_fixture_shape(
],
)
def test_run_pipeline_fail_invalid_destination_config(
- config: dict[str, Any], use_destination: str, tmp_path: Path
+ config: dict[str, Any], use_destination: str, tmp_path: Path, caplog: pytest.LogCaptureFixture
) -> None:
"""Invalid destination configs cause the pipeline to fail with an 'Invalid destination config' message."""
settings = CtsSettings(input_dir=tmp_path, output_dir=tmp_path, use_destination=use_destination) # pyright: ignore[reportCallIssue]
+ expected_msg = f"Invalid destination configuration for {use_destination}"
with (
isolated_dlt_config(config),
- pytest.raises(ValueError, match=f"Invalid destination configuration for {use_destination}"),
+ pytest.raises(ValueError, match=expected_msg),
):
run_pipeline(settings, [])
+ error_records = [r for r in caplog.records if r.name == core.__name__ and r.levelno == logging.ERROR]
+ assert len(error_records) == 1
+ assert error_records[0].getMessage() == expected_msg
+ assert error_records[0].exc_info is not None
+
@pytest.mark.parametrize(
("config", "use_destination"),
@@ -240,12 +246,16 @@ def test_run_pipeline_no_slack_env_var_when_vars_missing(
# run_pipeline tests
-def test_run_pipeline_minimal(dlt_test_settings: CtsSettings, destination_bucket: Path) -> None:
+def test_run_pipeline_minimal(
+ dlt_test_settings: CtsSettings, destination_bucket: Path, caplog: pytest.LogCaptureFixture
+) -> None:
"""A tiny pipeline loads its records into the destination bucket, not into output_dir."""
+ caplog.set_level(logging.INFO)
load_info = run_pipeline(dlt_test_settings, tiny_resource())
assert load_info is not None
assert not load_info.has_failed_jobs
+ assert str(load_info) in caplog.messages
assert read_output_jsonl_records(destination_bucket) == TINY_RESOURCE_DATA
assert read_destination_rows(Path(dlt_test_settings.output_dir), TINY_TABLE_NAME) == []
diff --git a/tests/cdm_data_loaders/pipelines/test_xml_common.py b/tests/cdm_data_loaders/pipelines/test_xml_common.py
new file mode 100644
index 00000000..1f21b141
--- /dev/null
+++ b/tests/cdm_data_loaders/pipelines/test_xml_common.py
@@ -0,0 +1,132 @@
+"""Unit tests for shared XML pipeline helpers in cdm_data_loaders.pipelines.xml_common."""
+
+import logging
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any
+
+import pyarrow as pa
+import pytest
+from lxml import etree
+
+from cdm_data_loaders.core.buffer import DictBuffer
+from cdm_data_loaders.core.registry import ArrowSchemaRegistry, TransformContext
+from cdm_data_loaders.pipelines.xml_common import (
+ _log_interval_final,
+ buffer_single_tag_element,
+ process_single_tag_xml_file,
+ stream_and_buffer_element_iter,
+)
+
+
+def _convert_id_one(_el: etree._Element) -> dict[str, str]:
+ return {"id": "1"}
+
+
+def _no_transform(_table: str) -> None:
+ return None
+
+
+def test_log_interval_final_logs_single_entry_when_not_on_boundary(
+ caplog: pytest.LogCaptureFixture,
+ tmp_path: Path,
+) -> None:
+ """Verify _log_interval_final logs when n_entries == 0 and (n_entries + 1) % log_interval != 0."""
+ xml_path = tmp_path / "one.xml"
+ xml_path.write_text("", encoding="utf-8")
+ with caplog.at_level(logging.DEBUG, logger="cdm_data_loaders.pipelines.xml_common"):
+ _log_interval_final(0, xml_path, 10)
+ assert "Processed 1 entries from one.xml" in caplog.text
+
+
+def test_buffer_single_tag_element_uses_buffer_schema_map(tmp_path: Path) -> None:
+ """Verify buffer_single_tag_element passes buffer.schema_map to route_and_coerce_record."""
+ schema = pa.schema([pa.field("id", pa.string()), pa.field("count", pa.int64())])
+ buf = DictBuffer(max_items=1, schema_map=ArrowSchemaRegistry(registry={"items": schema}))
+ el = etree.fromstring(b"- a7
")
+ ctx = TransformContext.for_source("items", str(tmp_path / "a.xml"))
+
+ def _convert(_el: etree._Element) -> dict[str, str]:
+ return {"id": "a", "count": "7"}
+
+ flushed = list(
+ buffer_single_tag_element(
+ el,
+ convert_element=_convert,
+ transform=None,
+ buffer=buf,
+ ctx=ctx,
+ )
+ )
+ assert len(flushed) == 1
+ table = flushed[0].data
+ assert isinstance(table, pa.Table)
+ assert table.schema.equals(schema)
+ assert table.to_pylist() == [{"id": "a", "count": 7}]
+
+
+def test_process_single_tag_xml_file_missing_target_raises(tmp_path: Path) -> None:
+ """Verify process_single_tag_xml_file raises RuntimeError when table_name is missing."""
+ xml_path = tmp_path / "ns.xml"
+ xml_path.write_text('- 1
', encoding="utf-8")
+ missing_settings = SimpleNamespace(table_name=None, xml_tag="item", log_interval=10, buffer_size=10)
+
+ with pytest.raises(RuntimeError, match=r"^Expected table_name and xml_tag when xml_level is not set$"):
+ list(process_single_tag_xml_file(xml_path, missing_settings, _convert_id_one))
+
+
+@pytest.mark.parametrize(
+ ("resolve_default_ns", "expected_rows"),
+ [
+ pytest.param(False, [], id="ignore-default-ns"),
+ pytest.param(True, [{"id": "1"}], id="resolve-default-ns"),
+ ],
+)
+def test_process_single_tag_xml_file_default_namespace(
+ tmp_path: Path,
+ resolve_default_ns: bool,
+ expected_rows: list[dict[str, str]],
+) -> None:
+ """Verify process_single_tag_xml_file matches namespaced tags only when resolve_default_ns is True."""
+ xml_path = tmp_path / "ns.xml"
+ xml_path.write_text('- 1
', encoding="utf-8")
+ schema = pa.schema([pa.field("id", pa.string())])
+ settings = SimpleNamespace(
+ table_name="items",
+ xml_tag="item",
+ log_interval=10,
+ buffer_size=10,
+ schema_map=ArrowSchemaRegistry(registry={"items": schema}),
+ use_pyarrow=True,
+ get_table_transform=_no_transform,
+ )
+
+ pages = list(
+ process_single_tag_xml_file(
+ xml_path,
+ settings,
+ _convert_id_one,
+ resolve_default_ns=resolve_default_ns,
+ )
+ )
+
+ rows = [row for page in pages for row in page.data.to_pylist()]
+ assert rows == expected_rows
+
+
+def test_stream_and_buffer_element_iter_flushes_and_logs(caplog: pytest.LogCaptureFixture, tmp_path: Path) -> None:
+ """Verify stream_and_buffer_element_iter iterates elements, logs progress, and flushes buffered items."""
+ xml_path = tmp_path / "items.xml"
+ xml_path.write_text("", encoding="utf-8")
+ buf = DictBuffer(max_items=10, use_pyarrow=False)
+ elements = [etree.fromstring(b" "), etree.fromstring(b" ")]
+
+ def _buffer_fn(_el: etree._Element) -> list[Any]:
+ return list(buf.add_items({"items": [{"id": "x"}]}))
+
+ with caplog.at_level(logging.DEBUG, logger="cdm_data_loaders.pipelines.xml_common"):
+ pages = list(stream_and_buffer_element_iter(xml_path, elements, 2, buf, _buffer_fn))
+
+ assert len(pages) == 1
+ assert pages[0].data == [{"id": "x"}, {"id": "x"}]
+ assert "Processed 2 entries" in caplog.text
diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py
index 0fb72485..737a2b71 100644
--- a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py
+++ b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py
@@ -152,16 +152,25 @@ def transform(row: dict) -> dict:
]
-def test_run_xmltodict_pipeline_uses_subclass_cli_prog_name(
+@pytest.mark.parametrize(
+ ("cli_prog_name", "expected_pipeline_name"),
+ [
+ pytest.param("preset_xmltodict", "preset_xmltodict", id="subclass-cli-prog-name"),
+ pytest.param(None, PIPELINE_NAME, id="unset-cli-prog-name-fallback"),
+ ],
+)
+def test_run_xmltodict_pipeline_resolves_pipeline_name(
settings_factory: Callable[..., XmlToDictSettings],
+ cli_prog_name: str | None,
+ expected_pipeline_name: str,
) -> None:
- """run_xmltodict_pipeline forwards a preset subclass cli_prog_name as pipeline_name."""
+ """run_xmltodict_pipeline uses subclass cli_prog_name or falls back to PIPELINE_NAME."""
- class PresetXmlToDictSettings(XmlToDictSettings, cli_prog_name="preset_xmltodict"):
+ class CustomXmlToDictSettings(XmlToDictSettings, cli_prog_name=cli_prog_name):
pass
base_settings = settings_factory()
- preset_settings = PresetXmlToDictSettings.model_validate(
+ custom_settings = CustomXmlToDictSettings.model_validate(
base_settings.model_dump(exclude=set(XmlToDictSettings.model_computed_fields))
)
- assert_core_run_pipeline_args(xmltodict_module, run_xmltodict_pipeline, preset_settings, "preset_xmltodict")
+ assert_core_run_pipeline_args(xmltodict_module, run_xmltodict_pipeline, custom_settings, expected_pipeline_name)
diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_processors.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_processors.py
index 90e6e6e2..3c547ca4 100644
--- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_processors.py
+++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_processors.py
@@ -3,6 +3,7 @@
import gzip
import logging
from collections.abc import Generator
+from datetime import date
from pathlib import Path
from typing import Final
from unittest.mock import MagicMock
@@ -26,7 +27,7 @@
)
from cdm_data_loaders.core.buffer import DictBuffer
from cdm_data_loaders.core.fields import BUFFER_SIZE, DEFAULTS, LOG_INTERVAL
-from cdm_data_loaders.core.registry import ArrowSchemaRegistry
+from cdm_data_loaders.core.registry import ArrowSchemaRegistry, TransformContext
from cdm_data_loaders.pipelines.xmltopyarrow.processors import process_xml_file_to_pyarrow
from cdm_data_loaders.pipelines.xmltopyarrow.settings import XmlToPyArrowSettings
from cdm_data_loaders.utils.recursion import DEFAULT_MAX_RECURSION_DEPTH
@@ -613,3 +614,78 @@ def test_process_xml_file_to_pyarrow_pass_use_pyarrow_false_yields_lists_of_dict
assert table_name == "people"
assert isinstance(data, list)
assert data == PEOPLE_XML_TO_PYARROW_PARSED[:2]
+
+
+def test_process_xml_file_to_pyarrow_pass_level_mode_applies_table_transform_and_source_file(
+ tmp_path: Path,
+) -> None:
+ """Level mode applies settings.get_table_transform(tag_name) and forwards source_file to TransformContext."""
+ file_path = write_xml_file(tmp_path, "catalog.xml", MULTI_ENTITY_XML)
+ settings = fake_xmltopyarrow_level_settings(xml_level=1)
+
+ def transform_book(row: dict[str, object], ctx: TransformContext) -> dict[str, list[dict[str, object]]]:
+ return {
+ "book": [{"_id": f"{row['_id']}:{Path(str(ctx.source_file)).name}", "title": str(row["title"]).upper()}],
+ }
+
+ settings.get_table_transform.side_effect = lambda name: transform_book if name == "book" else None
+ by_table = dict(table_and_data(process_xml_file_to_pyarrow(file_path, settings)))
+
+ assert by_table["book"].to_pylist() == [
+ {"_id": "b1:catalog.xml", "title": "DUNE"},
+ {"_id": "b2:catalog.xml", "title": "FOUNDATION"},
+ ]
+ assert by_table["magazine"].to_pylist() == [{"_id": "m1", "issue": 42}]
+
+
+def test_process_xml_file_to_pyarrow_pass_level_mode_coerces_transformed_records_against_schema_map(
+ tmp_path: Path,
+) -> None:
+ """Level mode forwards buffer.schema_map to route_and_coerce_record so string dates coerce for PyArrow."""
+ file_path = write_xml_file(tmp_path, "catalog.xml", MULTI_ENTITY_XML)
+ settings = fake_xmltopyarrow_level_settings(xml_level=1)
+ book_dates_schema = pa.schema([pa.field("_id", pa.string()), pa.field("published", pa.date32())])
+ assert settings.schema_map is not None
+ settings.schema_map = ArrowSchemaRegistry(
+ registry={**settings.schema_map.registry, "book_dates": book_dates_schema}
+ )
+
+ def transform_book(row: dict[str, object]) -> dict[str, list[dict[str, object]]]:
+ return {"book_dates": [{"_id": row["_id"], "published": "2026-01-15"}]}
+
+ settings.get_table_transform.side_effect = lambda name: transform_book if name == "book" else None
+ by_table = dict(table_and_data(process_xml_file_to_pyarrow(file_path, settings)))
+
+ assert by_table["book_dates"].schema.equals(book_dates_schema, check_metadata=True)
+ assert by_table["book_dates"].to_pylist() == [
+ {"_id": "b1", "published": date(2026, 1, 15)},
+ {"_id": "b2", "published": date(2026, 1, 15)},
+ ]
+
+
+@pytest.mark.parametrize(
+ ("missing_attr", "expected_msg"),
+ [
+ pytest.param(
+ "table_name", "Expected table_name and xml_tag when xml_level is not set", id="missing-table-name"
+ ),
+ pytest.param("xml_tag", "Expected table_name and xml_tag when xml_level is not set", id="missing-xml-tag"),
+ pytest.param(
+ "xml_element_converter",
+ "Expected xml_element_converter when xml_level is not set",
+ id="missing-converter",
+ ),
+ ],
+)
+def test_process_xml_file_to_pyarrow_fail_missing_single_tag_fields(
+ tmp_path: Path,
+ missing_attr: str,
+ expected_msg: str,
+) -> None:
+ """process_xml_file_to_pyarrow raises RuntimeError when single-tag mode is missing required attributes."""
+ file_path = write_xml_file(tmp_path, "two.xml", PEOPLE_XML_2)
+ settings = fake_xmltopyarrow_settings()
+ setattr(settings, missing_attr, None)
+
+ with pytest.raises(RuntimeError, match=expected_msg):
+ list(process_xml_file_to_pyarrow(file_path, settings))
diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py
index 196c0d13..8cbbbe53 100644
--- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py
+++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_pipeline.py
@@ -388,3 +388,18 @@ def test_pubmed_cli_pass_file_listing_skips_md5_sidecar_preprocessor(
"pubmed25n0001.xml.gz",
],
}
+
+
+def test_run_xmltopyarrow_pipeline_falls_back_to_pipeline_name_when_cli_prog_name_unset(
+ settings_factory: Callable[..., XmlToPyArrowSettings],
+) -> None:
+ """run_xmltopyarrow_pipeline falls back to PIPELINE_NAME when cli_prog_name is None."""
+
+ class NoProgXmlToPyArrowSettings(XmlToPyArrowSettings, cli_prog_name=None):
+ pass
+
+ base_settings = settings_factory()
+ no_prog_settings = NoProgXmlToPyArrowSettings.model_validate(
+ base_settings.model_dump(exclude=set(XmlToPyArrowSettings.model_computed_fields))
+ )
+ assert_core_run_pipeline_args(xmltopyarrow_module, run_xmltopyarrow_pipeline, no_prog_settings, PIPELINE_NAME)
diff --git a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py
index 301b7ef5..a7295950 100644
--- a/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py
+++ b/tests/cdm_data_loaders/pipelines/xmltopyarrow/test_xmltopyarrow_settings.py
@@ -8,6 +8,7 @@
import pyarrow as pa
import pytest
import xmlschema
+from lxml import etree
from pydantic import ValidationError
from pydantic_settings import CliApp, SettingsError
@@ -527,3 +528,121 @@ def test_xmltopyarrow_settings_pass_skips_pyarrow_schema_generation_for_jsonl_fo
assert settings.pyarrow_schemas == {}
assert gen_single_spy.call_count == 0
assert gen_level_spy.call_count == 0
+
+
+@pytest.mark.parametrize(
+ ("kwargs", "expected_message"),
+ [
+ pytest.param(
+ {"xml_level": 1, "xml_tag": "entry"},
+ "xml_level cannot be combined with xml_tag or table_name",
+ id="xml_level_with_xml_tag",
+ ),
+ pytest.param(
+ {"table_name": None, "xml_tag": None},
+ "Missing required field(s) when xml_level is not set: table_name, xml_tag",
+ id="missing_both_table_and_tag",
+ ),
+ pytest.param(
+ {"table_name": "entry", "xml_tag": "entry", "root_tag": "UniRef50"},
+ "root_tag can only be specified when xml_level is set",
+ id="root_tag_without_xml_level",
+ ),
+ ],
+)
+def test_xmltopyarrow_settings_validate_target_selection_exact_messages(
+ settings_factory: Callable[..., XmlToPyArrowSettings],
+ kwargs: dict[str, Any],
+ expected_message: str,
+) -> None:
+ """Target selection validation raises ValidationError with exact error messages."""
+ with pytest.raises(ValidationError) as exc_info:
+ settings_factory(xsd_file=str(PUBMED_XSD_FILE), **kwargs)
+ assert exc_info.value.errors()[0]["msg"] == f"Value error, {expected_message}"
+
+
+def test_xmltopyarrow_settings_level_mode_with_jsonl_validates_transforms_against_converters(
+ monkeypatch: pytest.MonkeyPatch,
+ settings_factory: Callable[..., XmlToPyArrowSettings],
+) -> None:
+ """In level mode with use_pyarrow=False, table validation uses xml_level_converters keys."""
+ transforms_mod = register_temp_module(
+ monkeypatch,
+ "_test_xmltopyarrow_level_jsonl_transforms",
+ TABLE_TRANSFORMS={"PubmedArticle": lambda r: {"PubmedArticle": [r]}},
+ )
+ settings = settings_factory(
+ xsd_file=str(PUBMED_XSD_FILE),
+ xml_level=1,
+ root_tag=PUBMED_ROOT_TAG,
+ loader_file_format="jsonl",
+ transforms_module=transforms_mod,
+ )
+ assert settings.pyarrow_schemas == {}
+ assert "PubmedArticle" in settings.xml_level_converters
+
+
+@pytest.mark.parametrize(
+ ("depth", "type_predicate", "expected_child_type"),
+ [
+ pytest.param(1, pa.types.is_string, str, id="depth-1-string"),
+ pytest.param(2, pa.types.is_struct, dict, id="depth-2-struct"),
+ ],
+)
+def test_xmltopyarrow_settings_level_mode_forwards_max_recursion_depth(
+ tmp_path: Path,
+ settings_factory: Callable[..., XmlToPyArrowSettings],
+ depth: int,
+ type_predicate: Callable[[pa.DataType], bool],
+ expected_child_type: type[object],
+) -> None:
+ """In level mode, max_recursion_depth controls both converter and PyArrow schema recursion depth."""
+ recursive_xsd = tmp_path / "recursive.xsd"
+ recursive_xsd.write_text(
+ """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+""",
+ encoding="utf-8",
+ )
+ settings = settings_factory(xsd_file=str(recursive_xsd), xml_level=1, root_tag="root", max_recursion_depth=depth)
+
+ assert type_predicate(settings.pyarrow_schemas["node"].field("child").type)
+ xml_el = etree.fromstring(b"ab")
+ row = settings.xml_level_converters["node"](xml_el)
+ assert isinstance(row["child"], expected_child_type)
+
+
+def test_preset_xmltopyarrow_settings_bind_cdm_transform_passes_default_pyarrow_schema(
+ settings_factory: Callable[..., XmlToPyArrowSettings],
+ copy_xsd_fixture: Callable[[str], str],
+) -> None:
+ """_bind_cdm_transform_and_schemas and _resolve_settings_schema_map honor default_pyarrow_schema."""
+ custom_schema = pa.schema([pa.field("custom_col", pa.string())])
+
+ class CustomPresetSettings(PresetXmlToPyArrowSettings, cli_prog_name="custom_preset"):
+ PRECOMPUTED_SCHEMA: ClassVar[pa.Schema] = pa.schema([pa.field("entry_col", pa.string())])
+
+ def _resolve_settings_schema_map(self, default_pyarrow_schema: pa.Schema | None = None) -> None:
+ self._bind_cdm_transform_and_schemas(
+ lambda r: {"entry": [r]},
+ ArrowSchemaRegistry(registry={"entry": custom_schema}),
+ default_pyarrow_schema=custom_schema if default_pyarrow_schema is None else default_pyarrow_schema,
+ )
+
+ xsd_path = copy_xsd_fixture(UNIREF_LIKE_XSD)
+ settings = settings_factory(CustomPresetSettings, dataset_name="custom_ds", xsd_file=xsd_path)
+ assert settings.pyarrow_schema is custom_schema
diff --git a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py
index c0dd96c9..69ae0644 100644
--- a/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py
+++ b/tests/cdm_data_loaders/pipelines/xsv/test_xsv_pipeline.py
@@ -4,6 +4,7 @@
skipped when qsv is not available on PATH or via the QSV_BIN env var.
"""
+import gzip
import sys
import threading
from collections.abc import Callable
@@ -527,3 +528,89 @@ def fail_transform(_row: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
"top.csv",
],
}
+
+
+def test_process_xsv_file_pass_gzipped_csv(
+ settings_factory: Callable[..., XsvIngestSettings],
+ work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths],
+) -> None:
+ """Valid .csv.gz files are decompressed and loaded into the target table."""
+ settings = settings_factory(SIMPLE_SCHEMA, {})
+ work_paths = work_paths_factory(settings)
+
+ valid_gz = Path(settings.input_dir) / "good.csv.gz"
+ valid_gz.write_bytes(gzip.compress(b"number,name\n1,alice\n2,bob\n"))
+ results = list(_process_xsv_file(make_file_item(valid_gz), settings, work_paths))
+ assert len(results) == 1
+ assert results[0].meta.table_name == "my_table"
+ assert results[0].data.to_pylist() == [
+ {"number": 1, "name": "alice"},
+ {"number": 2, "name": "bob"},
+ ]
+
+
+def test_process_xsv_file_fail_corrupt_gzipped_csv_routes_to_rejected(
+ settings_factory: Callable[..., XsvIngestSettings],
+ work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths],
+) -> None:
+ """Corrupt .csv.gz files route an error record to _rejected."""
+ settings = settings_factory(SIMPLE_SCHEMA, {})
+ work_paths = work_paths_factory(settings)
+
+ bad_gz = Path(settings.input_dir) / "corrupt.csv.gz"
+ bad_gz.write_bytes(b"not-a-valid-gzip-stream")
+ results = list(_process_xsv_file(make_file_item(bad_gz), settings, work_paths))
+ assert [r.meta.table_name for r in results] == [REJECTED_TABLE_NAME]
+ (rejected,) = results[0].data.to_pylist()
+ assert rejected["table_name"] == "my_table"
+ assert rejected["source_file"] == "corrupt.csv.gz"
+ assert rejected["error_detail"]
+
+
+def test_process_xsv_file_pass_pyarrow_table_transform_routes_table_and_dict_outputs(
+ settings_factory: Callable[..., XsvIngestSettings],
+ work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths],
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """PyArrow table transforms route pa.Table outputs directly and dict-list outputs through DictBuffer."""
+
+ def split_table(table: pa.Table) -> dict[str, pa.Table | list[dict[str, Any]]]:
+ return {
+ "my_table": table,
+ "my_table_summary": [{"count": table.num_rows}],
+ }
+
+ transforms_mod = register_temp_module(
+ monkeypatch,
+ "_test_xsv_pyarrow_transforms",
+ TABLE_TRANSFORMS={"my_table": split_table},
+ )
+ settings = settings_factory(
+ SIMPLE_SCHEMA,
+ {"data.csv": "number,name\n1,alice\n2,bob\n"},
+ transforms_module=transforms_mod,
+ )
+ work_paths = work_paths_factory(settings)
+
+ results = list(_process_xsv_file(make_file_item(Path(settings.input_dir) / "data.csv"), settings, work_paths))
+ by_table = {r.meta.table_name: r.data for r in results}
+
+ assert set(by_table) == {"my_table", "my_table_summary"}
+ assert isinstance(by_table["my_table"], pa.Table)
+ assert by_table["my_table"].to_pylist() == [
+ {"number": 1, "name": "alice"},
+ {"number": 2, "name": "bob"},
+ ]
+ assert by_table["my_table_summary"] == [{"count": 2}]
+
+
+def test_process_xsv_file_pass_empty_csv_produces_no_rows(
+ settings_factory: Callable[..., XsvIngestSettings],
+ work_paths_factory: Callable[[XsvIngestSettings], XsvWorkPaths],
+) -> None:
+ """A completely empty CSV yields no rows and no errors."""
+ settings = settings_factory(SIMPLE_SCHEMA, {"empty.csv": ""})
+ work_paths = work_paths_factory(settings)
+
+ results = list(_process_xsv_file(make_file_item(Path(settings.input_dir) / "empty.csv"), settings, work_paths))
+ assert results == []
diff --git a/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py b/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py
index 85754ced..dead4320 100644
--- a/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py
+++ b/tests/cdm_data_loaders/readers/xml/test_xml_stream_xml_file.py
@@ -5,9 +5,15 @@
from pathlib import Path
import pytest
-from lxml.etree import XMLSyntaxError, tostring
-
-from cdm_data_loaders.readers.xml import stream_xml_file, stream_xml_file_at_level
+from lxml.etree import XMLSyntaxError, fromstring, tostring
+
+from cdm_data_loaders.readers.xml import (
+ clear_xml_element,
+ resolve_stream_tag,
+ stream_elements,
+ stream_xml_file,
+ stream_xml_file_at_level,
+)
from tests.helpers.fs_backends import FsBackend
from tests.helpers.xml_samples import SIMPLE_LIBRARY_XML, XML_OPEN_ERROR_CASES, prepare_xml_error_path
@@ -336,3 +342,110 @@ def test_stream_xml_file_at_level_fail_negative_level(xml_path_factory: Callable
path = xml_path_factory(SIMPLE_LIBRARY_XML)
with pytest.raises(ValueError, match="level must be >= 0"):
list(stream_xml_file_at_level(path, -1))
+
+
+def test_clear_xml_element_pass_prunes_preceding_siblings() -> None:
+ """clear_xml_element clears the target element and removes its preceding siblings."""
+ root = fromstring(b"text")
+ b_elem = root[1]
+ clear_xml_element(b_elem)
+ assert [child.tag for child in root] == ["b", "c"]
+ assert b_elem.text is None
+
+
+def test_clear_xml_element_pass_ignores_non_element() -> None:
+ """clear_xml_element ignores values that are not lxml elements."""
+ clear_xml_element("not-an-element") # type: ignore[arg-type]
+
+
+@pytest.mark.parametrize(
+ ("xml", "filename", "tag", "expected", "expect_unread"),
+ [
+ pytest.param(
+ NAMESPACED_SIMPLE_XML,
+ "ns.xml",
+ f"{{{NAMESPACE_URI}}}book",
+ f"{{{NAMESPACE_URI}}}book",
+ True,
+ id="already-qualified",
+ ),
+ pytest.param(
+ NAMESPACED_SIMPLE_XML,
+ "ns.xml",
+ "book",
+ f"{{{NAMESPACE_URI}}}book",
+ False,
+ id="default-namespace-unprefixed",
+ ),
+ pytest.param(
+ SIMPLE_LIBRARY_XML,
+ "plain.xml",
+ "book",
+ "book",
+ False,
+ id="no-namespace-unprefixed",
+ ),
+ ],
+)
+def test_resolve_stream_tag_pass_default_namespace(
+ xml_path_factory: Callable[..., Path],
+ xml: str,
+ filename: str,
+ tag: str,
+ expected: str,
+ expect_unread: bool,
+) -> None:
+ """resolve_stream_tag qualifies unprefixed tags from the default namespace without reading pre-qualified tags."""
+ path = xml_path_factory(xml, filename=filename)
+ with path.open("rb") as fh:
+ assert resolve_stream_tag(fh, tag) == expected
+ if expect_unread:
+ assert fh.tell() == 0
+
+
+@pytest.mark.parametrize(
+ ("tag", "resolve_default_ns", "expected"),
+ [
+ pytest.param(
+ "book",
+ True,
+ [(f"{{{NAMESPACE_URI}}}book", "1"), (f"{{{NAMESPACE_URI}}}book", "2"), (f"{{{NAMESPACE_URI}}}book", "3")],
+ id="unprefixed-resolve-enabled",
+ ),
+ pytest.param("book", False, [], id="unprefixed-resolve-disabled"),
+ pytest.param(
+ f"{{{NAMESPACE_URI}}}book",
+ True,
+ [(f"{{{NAMESPACE_URI}}}book", "1"), (f"{{{NAMESPACE_URI}}}book", "2"), (f"{{{NAMESPACE_URI}}}book", "3")],
+ id="qualified-resolve-enabled",
+ ),
+ ],
+)
+def test_stream_elements_pass_default_namespace_resolution(
+ xml_path_factory: Callable[..., Path],
+ tag: str,
+ resolve_default_ns: bool,
+ expected: list[tuple[str, str | None]],
+) -> None:
+ """stream_elements resolves unprefixed tags against the default namespace only when resolve_default_ns is True."""
+ ns_path = xml_path_factory(NAMESPACED_SIMPLE_XML, filename="ns.xml")
+ actual = [(el.tag, el.get("id")) for el in stream_elements(ns_path, tag, resolve_default_ns=resolve_default_ns)]
+ assert actual == expected
+
+
+def test_stream_xml_file_at_level_pass_prunes_intermediate_ancestors(
+ xml_path_factory: Callable[..., Path],
+) -> None:
+ """stream_xml_file_at_level yields target depth elements while clearing intermediate ancestors."""
+ path = xml_path_factory("\n \n 1\n 2\n \n")
+ elems = list(stream_xml_file_at_level(path, 2))
+ assert [el.tag for el in elems] == ["b", "c"]
+
+
+def test_stream_xml_file_at_level_pass_removes_blank_text(
+ xml_path_factory: Callable[..., Path],
+) -> None:
+ """stream_xml_file_at_level strips indentation whitespace between child elements."""
+ path = xml_path_factory(SIMPLE_LIBRARY_XML)
+ first_book = next(stream_xml_file_at_level(path, 1))
+ assert first_book.text is None
diff --git a/tests/cdm_data_loaders/readers/xsv/test_duckdb_reader.py b/tests/cdm_data_loaders/readers/xsv/test_duckdb_reader.py
index c65d5f4f..c743db27 100644
--- a/tests/cdm_data_loaders/readers/xsv/test_duckdb_reader.py
+++ b/tests/cdm_data_loaders/readers/xsv/test_duckdb_reader.py
@@ -8,6 +8,7 @@
import pytest
from cdm_data_loaders.readers.xsv.duckdb_reader import (
+ _read_header_columns,
read_validated_rows,
resolve_duckdb_column_config,
resolve_xsv_parsing_config,
@@ -247,3 +248,53 @@ def test_read_validated_rows_pass_emits_pyarrow_tables_without_pa_schema(tmp_pat
assert len(tables) == 1
assert isinstance(tables[0], pa.Table)
assert tables[0].to_pylist() == [{"a": "1", "b": "foo"}, {"a": "2", "b": "bar"}]
+
+
+def test_read_validated_rows_pass_defaults_to_tab_and_respects_chunk_size(tmp_path: Path) -> None:
+ """read_validated_rows defaults to tab delimiter and accepts chunk_size in dict mode."""
+ pa_schema = pa.schema(
+ [
+ pa.field("a", pa.int64(), nullable=False),
+ pa.field("b", pa.string(), nullable=False),
+ ]
+ )
+ xsv_path = write_xsv(tmp_path, "a\tb\n1\tfoo\n2\tbar\n", file_name="data.tsv")
+ rows = list(read_validated_rows(xsv_path, pa_schema, chunk_size=1))
+ assert rows == [{"a": 1, "b": "foo"}, {"a": 2, "b": "bar"}]
+
+
+@pytest.mark.parametrize(
+ ("content", "file_name", "delimiter", "quote", "escape", "expected"),
+ [
+ pytest.param(
+ '\n\n"a,1", b \n1,2\n',
+ "quoted.csv",
+ ",",
+ '"',
+ None,
+ {"a,1", "b"},
+ id="blank-lines-quoted-and-padded",
+ ),
+ pytest.param(
+ '"a\\"1",b\n1,2\n',
+ "escaped.csv",
+ ",",
+ '"',
+ "\\",
+ {'a"1', "b"},
+ id="escaped-quote",
+ ),
+ ],
+)
+def test_read_header_columns_pass_quoted_escaped_and_padded(
+ tmp_path: Path,
+ content: str,
+ file_name: str,
+ delimiter: str,
+ quote: str,
+ escape: str | None,
+ expected: set[str],
+) -> None:
+ """_read_header_columns skips blank lines, respects quote and escape, and strips column names."""
+ path = write_xsv(tmp_path, content, file_name=file_name)
+ assert _read_header_columns(path, delimiter, quote, escape) == expected
diff --git a/tests/cdm_data_loaders/readers/xsv/test_qsv.py b/tests/cdm_data_loaders/readers/xsv/test_qsv.py
index 4e1cc7af..e8110c14 100644
--- a/tests/cdm_data_loaders/readers/xsv/test_qsv.py
+++ b/tests/cdm_data_loaders/readers/xsv/test_qsv.py
@@ -1966,3 +1966,62 @@ def test_clean_validate_file_pass_end_to_end_dereferences_ref_and_allof_integer_
assert "$defs" not in written_qsv_schema
assert written_qsv_schema["properties"]["reads"]["type"] == ["integer", "null"]
assert parse_xsv((args.validated_file_dir_path / result).read_text()) == [["gene_id", "reads"], *rows]
+
+
+@pytest.mark.parametrize("use_env", [False, True], ids=["from-path", "from-qsv-bin-env"])
+def test_qsv_check_pass_resolves_binary_from_path_or_env(
+ monkeypatch: pytest.MonkeyPatch,
+ qsv_cmd: str,
+ use_env: bool,
+) -> None:
+ """qsv_check resolves the binary from PATH or falls back to QSV_BIN."""
+ if use_env:
+ monkeypatch.setattr(shutil, "which", lambda _name: None)
+ monkeypatch.setenv("QSV_BIN", qsv_cmd)
+ assert qsv_check() == qsv_cmd
+
+
+@pytest.mark.parametrize(
+ ("qsv_bin", "match"),
+ [
+ pytest.param(None, "Could not locate the qsv binary", id="missing-qsv-bin"),
+ pytest.param("/nonexistent/qsv-binary", "Cannot perform validation with qsv:", id="nonexistent-qsv-bin"),
+ ],
+)
+def test_qsv_check_fail_missing_or_invalid_qsv_bin(
+ monkeypatch: pytest.MonkeyPatch,
+ qsv_bin: str | None,
+ match: str,
+) -> None:
+ """qsv_check raises RuntimeError when the binary is absent or fails to execute."""
+ monkeypatch.setattr(shutil, "which", lambda _name: None)
+ if qsv_bin is None:
+ monkeypatch.delenv("QSV_BIN", raising=False)
+ else:
+ monkeypatch.setenv("QSV_BIN", qsv_bin)
+ with pytest.raises(RuntimeError, match=match):
+ qsv_check()
+
+
+def test_run_qsv_validate_pass_preserves_non_xsv_schema_dialect(
+ make_args: Callable[..., CleanerValidatorArgs],
+ write_source_file: WriteFile,
+ write_working_file: WriteFile,
+) -> None:
+ """run_qsv_validate preserves standard $schema URIs when has_xsv_metaschema is False."""
+ draft7_uri = "http://json-schema.org/draft-07/schema#"
+ draft7_schema = ValidatedXsvSchema(
+ jsonschema={
+ "$schema": draft7_uri,
+ "type": "object",
+ "required": ["gene_id"],
+ "properties": {"gene_id": {"type": "string"}},
+ }
+ )
+ source = write_source_file("gene_id\nG0001\n", "genes.tsv")
+ args = make_args(source, first_pass_schema_override=draft7_schema)
+ input_name = _write_validator_input_file(args, write_working_file, [["G0001"]], "-with-headers")
+ run_qsv_validate(args, input_name, draft7_schema, first_pass=True)
+ files = generate_qsv_validate_file_names(args, input_name, first_pass=True)
+ written = json.loads(files.schema.read_text(encoding="utf-8"))
+ assert written["$schema"] == draft7_uri
diff --git a/tests/cdm_data_loaders/utils/iceberg/test_cleanup.py b/tests/cdm_data_loaders/utils/iceberg/test_cleanup.py
index d3d5e35a..e13174ec 100644
--- a/tests/cdm_data_loaders/utils/iceberg/test_cleanup.py
+++ b/tests/cdm_data_loaders/utils/iceberg/test_cleanup.py
@@ -3,9 +3,11 @@
import gc
import importlib
import logging
+import os
import sys
import threading
import warnings
+from pathlib import Path
from types import TracebackType
from typing import Final
from unittest.mock import MagicMock, patch
@@ -15,6 +17,7 @@
from dlt.common.libs import pyiceberg as dlt_pyiceberg
from dlt.common.libs.pyiceberg import get_iceberg_tables
from dlt.destinations.impl.filesystem.filesystem import FilesystemClient
+from pyiceberg import catalog as pyiceberg_cat
from pyiceberg.catalog import sql as pyiceberg_sql
from pyiceberg.catalog.rest import RestCatalog
from pyiceberg.catalog.sql import SqlCatalog
@@ -28,6 +31,7 @@
from cdm_data_loaders.pipelines.runner import run_pipeline
from cdm_data_loaders.utils.iceberg.cleanup import (
DLT_LOAD_CATALOG_ATTR,
+ PYICEBERG_ENV_CONFIG_ATTR,
install_iceberg_catalog_cleanup,
)
@@ -229,6 +233,214 @@ def test_iceberg_catalog_cleanup_fail_sql_catalog_integrity_error_becomes_namesp
with (
SqlCatalog("dup_ns", uri="sqlite:///:memory:") as catalog,
patch("sqlalchemy.orm.Session.commit", side_effect=IntegrityError("INSERT", {}, Exception("UNIQUE"))),
- pytest.raises(NamespaceAlreadyExistsError, match="Namespace ns_two already exists"),
+ pytest.raises(NamespaceAlreadyExistsError, match="Namespace ns_two already exists") as exc_info,
):
catalog.create_namespace("ns_two", {"owner": "cdm"})
+ assert isinstance(exc_info.value.__cause__, IntegrityError)
+
+
+def _fresh_install_cleanup(monkeypatch: pytest.MonkeyPatch) -> object:
+ """Reload cleanup module in isolation and install hooks."""
+ for target, attr in PATCH_TARGETS:
+ monkeypatch.setattr(target, attr, getattr(target, attr))
+ monkeypatch.delitem(sys.modules, "cdm_data_loaders.utils.iceberg.cleanup", raising=False)
+ monkeypatch.delattr(iceberg_pkg, "cleanup", raising=False)
+ fresh_cleanup = importlib.import_module("cdm_data_loaders.utils.iceberg.cleanup")
+ fresh_cleanup.install_iceberg_catalog_cleanup()
+ return fresh_cleanup
+
+
+def test_iceberg_catalog_cleanup_pass_rest_catalog_close_invokes_session_close() -> None:
+ """RestCatalog.close() invokes session.close() via the wrapped close hook."""
+ install_iceberg_catalog_cleanup()
+ mock_session = MagicMock(spec=Session)
+ with (
+ patch.object(RestCatalog, "_fetch_config"),
+ patch.object(RestCatalog, "_create_session", return_value=mock_session),
+ ):
+ catalog = RestCatalog("polaris", uri="http://polaris:8181/api/catalog")
+ catalog.close()
+ count_after_first_close = mock_session.close.call_count
+ assert count_after_first_close > 0
+ catalog.close()
+ assert mock_session.close.call_count == count_after_first_close + 1
+
+
+def test_iceberg_catalog_cleanup_pass_rest_catalog_without_session_has_no_finalizer() -> None:
+ """RestCatalog skips registering a finalizer when _create_session returns None."""
+ install_iceberg_catalog_cleanup()
+ with (
+ patch.object(RestCatalog, "_fetch_config"),
+ patch.object(RestCatalog, "_create_session", return_value=None),
+ ):
+ no_session_catalog = RestCatalog("polaris_none", uri="http://polaris:8181/api/catalog")
+ assert not hasattr(no_session_catalog, "_finalizer")
+
+
+@pytest.mark.parametrize(
+ ("url", "call_kwargs", "expected_kwargs"),
+ [
+ pytest.param("sqlite:///a.db", {}, {"connect_args": {"check_same_thread": False}}, id="sqlite-default"),
+ pytest.param(
+ "sqlite:///b.db",
+ {"connect_args": {"timeout": 10}},
+ {"connect_args": {"timeout": 10, "check_same_thread": False}},
+ id="sqlite-preserves-other-connect-args",
+ ),
+ pytest.param(
+ "sqlite:///c.db",
+ {"connect_args": {"check_same_thread": True}},
+ {"connect_args": {"check_same_thread": True}},
+ id="sqlite-preserves-explicit-check-same-thread",
+ ),
+ pytest.param("postgresql://localhost/db", {}, {}, id="non-sqlite-unchanged"),
+ ],
+)
+def test_iceberg_catalog_cleanup_pass_create_engine_connect_args(
+ monkeypatch: pytest.MonkeyPatch,
+ url: str,
+ call_kwargs: dict[str, object],
+ expected_kwargs: dict[str, object],
+) -> None:
+ """Patched create_engine sets check_same_thread=False only for sqlite URLs and preserves existing args."""
+ recorded: list[tuple[object, dict[str, object]]] = []
+
+ def fake_create_engine(engine_url: object, *_args: object, **kwargs: object) -> object:
+ recorded.append((engine_url, dict(kwargs)))
+ return object()
+
+ monkeypatch.setattr(pyiceberg_sql, "create_engine", fake_create_engine)
+ _fresh_install_cleanup(monkeypatch)
+
+ pyiceberg_sql.create_engine(url, **call_kwargs)
+ assert recorded == [(url, expected_kwargs)]
+
+
+@pytest.mark.parametrize(
+ ("catalog_name", "create_yaml", "initial_env_home", "expect_tmp_path_home"),
+ [
+ pytest.param("cat1", False, None, False, id="no-yaml-no-env"),
+ pytest.param("cat2", True, None, True, id="home-yaml-sets-and-restores-env"),
+ pytest.param("cat3", True, "/custom/home", False, id="existing-env-preserved"),
+ ],
+)
+def test_iceberg_catalog_cleanup_pass_load_catalog_env_and_home(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ catalog_name: str,
+ create_yaml: bool,
+ initial_env_home: str | None,
+ expect_tmp_path_home: bool,
+) -> None:
+ """Patched _load_catalog_from_pyiceberg resets _ENV_CONFIG and temporarily sets PYICEBERG_HOME."""
+ seen_env_home: list[str | None] = []
+ seen_env_configs: list[object] = []
+
+ def fake_orig_load(name: str) -> str:
+ seen_env_home.append(os.environ.get("PYICEBERG_HOME"))
+ seen_env_configs.append(getattr(pyiceberg_cat, PYICEBERG_ENV_CONFIG_ATTR))
+ return f"loaded:{name}"
+
+ monkeypatch.setattr(dlt_pyiceberg, DLT_LOAD_CATALOG_ATTR, fake_orig_load)
+ _fresh_install_cleanup(monkeypatch)
+
+ monkeypatch.setattr(Path, "home", classmethod(lambda _cls: tmp_path))
+ if create_yaml:
+ (tmp_path / ".pyiceberg.yaml").write_text("catalog: {}\n")
+ if initial_env_home is None:
+ monkeypatch.delenv("PYICEBERG_HOME", raising=False)
+ else:
+ monkeypatch.setenv("PYICEBERG_HOME", initial_env_home)
+
+ sentinel_cfg = object()
+ monkeypatch.setattr(pyiceberg_cat, PYICEBERG_ENV_CONFIG_ATTR, sentinel_cfg)
+
+ result = getattr(dlt_pyiceberg, DLT_LOAD_CATALOG_ATTR)(catalog_name)
+ expected_seen_home = str(tmp_path) if expect_tmp_path_home else initial_env_home
+ assert result == f"loaded:{catalog_name}"
+ assert seen_env_home == [expected_seen_home]
+ assert seen_env_configs[0] is not sentinel_cfg
+ assert os.environ.get("PYICEBERG_HOME") == initial_env_home
+
+
+def test_iceberg_catalog_cleanup_fail_load_catalog_restores_pyiceberg_home_on_error(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Patched _load_catalog_from_pyiceberg clears temporary PYICEBERG_HOME even when loading raises."""
+
+ def fake_orig_load(_name: str) -> str:
+ msg = "load error"
+ raise ValueError(msg)
+
+ monkeypatch.setattr(dlt_pyiceberg, DLT_LOAD_CATALOG_ATTR, fake_orig_load)
+ _fresh_install_cleanup(monkeypatch)
+ monkeypatch.setattr(Path, "home", classmethod(lambda _cls: tmp_path))
+ monkeypatch.delenv("PYICEBERG_HOME", raising=False)
+ (tmp_path / ".pyiceberg.yaml").write_text("catalog: {}\n")
+
+ with pytest.raises(ValueError, match="load error"):
+ getattr(dlt_pyiceberg, DLT_LOAD_CATALOG_ATTR)("boom")
+ assert "PYICEBERG_HOME" not in os.environ
+
+
+@pytest.mark.parametrize(
+ ("catalog", "exc_args"),
+ [
+ pytest.param(None, (None, None, None), id="none-catalog-normal-exit"),
+ pytest.param(object(), (RuntimeError, RuntimeError("e"), None), id="non-closable-catalog-error-exit"),
+ ],
+)
+def test_iceberg_catalog_cleanup_pass_filesystem_client_exit_without_closable_catalog(
+ monkeypatch: pytest.MonkeyPatch,
+ catalog: object,
+ exc_args: tuple[type[BaseException] | None, BaseException | None, TracebackType | None],
+) -> None:
+ """FilesystemClient.__exit__ clears namespaces and delegates to orig __exit__ for None or non-closable catalogs."""
+ orig_exits: list[tuple[object, object, object]] = []
+
+ def fake_orig_exit(
+ _self: FilesystemClient,
+ exc_type: type[BaseException] | None,
+ exc_val: BaseException | None,
+ exc_tb: TracebackType | None,
+ ) -> None:
+ orig_exits.append((exc_type, exc_val, exc_tb))
+
+ monkeypatch.setattr(FilesystemClient, "__exit__", fake_orig_exit)
+ _fresh_install_cleanup(monkeypatch)
+
+ client = object.__new__(FilesystemClient)
+ client._catalog = catalog
+ client._catalog_namespaces = {"ns"}
+ client.__exit__(*exc_args)
+ assert client._catalog is None
+ assert client._catalog_namespaces == set()
+ assert orig_exits == [exc_args]
+
+
+def test_iceberg_catalog_cleanup_fail_filesystem_client_exit_calls_orig_exit_on_close_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """FilesystemClient.__exit__ still calls orig __exit__ when catalog.close() raises."""
+ orig_exits: list[tuple[object, object, object]] = []
+
+ def fake_orig_exit(
+ _self: FilesystemClient,
+ exc_type: type[BaseException] | None,
+ exc_val: BaseException | None,
+ exc_tb: TracebackType | None,
+ ) -> None:
+ orig_exits.append((exc_type, exc_val, exc_tb))
+
+ monkeypatch.setattr(FilesystemClient, "__exit__", fake_orig_exit)
+ _fresh_install_cleanup(monkeypatch)
+
+ failing_cat = MagicMock()
+ failing_cat.close.side_effect = RuntimeError("close failed")
+ client = object.__new__(FilesystemClient)
+ client._catalog = failing_cat
+ client._catalog_namespaces = {"ns"}
+ with pytest.raises(RuntimeError, match="close failed"):
+ client.__exit__(None, None, None)
+ assert orig_exits == [(None, None, None)]
diff --git a/tests/cdm_data_loaders/utils/test_arrow_schema.py b/tests/cdm_data_loaders/utils/test_arrow_schema.py
index ee4979c2..047d83fc 100644
--- a/tests/cdm_data_loaders/utils/test_arrow_schema.py
+++ b/tests/cdm_data_loaders/utils/test_arrow_schema.py
@@ -42,7 +42,7 @@ def test_assemble_pyarrow_schema_pass_from_struct_and_sequence(
def test_assemble_pyarrow_schema_rejects_empty_component() -> None:
"""Raise ``ValueError`` when assembling a schema with zero fields."""
- with pytest.raises(ValueError, match="Cannot assemble a PyArrow schema from an empty component"):
+ with pytest.raises(ValueError, match=r"^Cannot assemble a PyArrow schema from an empty component$"):
assemble_pyarrow_schema([])
diff --git a/tests/cdm_data_loaders/utils/test_logging.py b/tests/cdm_data_loaders/utils/test_logging.py
index 74656426..7084138d 100644
--- a/tests/cdm_data_loaders/utils/test_logging.py
+++ b/tests/cdm_data_loaders/utils/test_logging.py
@@ -378,6 +378,23 @@ def test_init_logger_pass_same_mtime_skipped_unless_forced(tmp_path: Path) -> No
assert root.handlers[0] not in handlers
+def test_init_logger_pass_reapplies_when_switching_config_files_with_same_mtime(tmp_path: Path) -> None:
+ """Switching to a different config file with the same mtime reapplies logging configuration."""
+ config_a = write_json_or_yaml(tmp_path / "logging_a.yaml", make_config("INFO"))
+ config_b = write_json_or_yaml(tmp_path / "logging_b.yaml", make_config("DEBUG"))
+ set_mtime(config_b, config_a.stat().st_mtime_ns)
+
+ init_logger(settings_for(config_a))
+ root = logging.getLogger()
+ handlers = list(root.handlers)
+ assert root.level == logging.INFO
+
+ init_logger(settings_for(config_b))
+ assert root.level == logging.DEBUG
+ assert len(root.handlers) == 1
+ assert root.handlers[0] not in handlers
+
+
def test_init_logger_pass_force_relogs_fallback_warning(capsys: pytest.CaptureFixture[str]) -> None:
"""Forcing the fallback config rebuilds the handler and logs the warning again."""
init_logger(settings_for(None))
diff --git a/tests/cdm_data_loaders/utils/test_progress.py b/tests/cdm_data_loaders/utils/test_progress.py
index f71c3eed..b0899c30 100644
--- a/tests/cdm_data_loaders/utils/test_progress.py
+++ b/tests/cdm_data_loaders/utils/test_progress.py
@@ -292,14 +292,21 @@ def test_make_progress_bar_pass_update_advances_n_regardless_of_disable(disable:
def test_make_progress_bar_fail_requires_keyword_arguments() -> None:
"""All parameters are keyword-only; calling with positional arguments raises TypeError."""
- with pytest.raises(TypeError, match=r"make_progress_bar\(\) takes 0 positional arguments but 2 were given"):
+ with pytest.raises(TypeError, match=r"make_progress_bar.*? takes 0 positional arguments but 2 were given"):
make_progress_bar(100, "test") # type: ignore[misc,call-arg]
-@pytest.mark.parametrize("args", [{}, {"total": 10}, {"desc": "test"}])
+@pytest.mark.parametrize(
+ "args",
+ [
+ pytest.param({}, id="missing-both"),
+ pytest.param({"total": 10}, id="missing-desc"),
+ pytest.param({"desc": "test"}, id="missing-total"),
+ ],
+)
def test_make_progress_bar_fail_missing_required_arguments(args: dict[str, int | str]) -> None:
"""Keywords 'total' and 'desc' are required; omitting them raises TypeError."""
- regex = re.compile(r"make_progress_bar\(\) missing [12] required keyword-only arguments?")
+ regex = re.compile(r"make_progress_bar.*? missing [12] required keyword-only arguments?")
with pytest.raises(TypeError, match=regex):
make_progress_bar(**args) # type: ignore[call-arg]
diff --git a/tests/cdm_data_loaders/validation/jsonschema/test_dereference.py b/tests/cdm_data_loaders/validation/jsonschema/test_dereference.py
index df10d589..96730c7e 100644
--- a/tests/cdm_data_loaders/validation/jsonschema/test_dereference.py
+++ b/tests/cdm_data_loaders/validation/jsonschema/test_dereference.py
@@ -13,6 +13,7 @@
RefSiblingMode,
_dereference,
_merge_all_of,
+ _merge_schema_pair,
_resolve_ref_and_all_of,
_should_apply_ref_siblings,
dereference_schema,
@@ -874,3 +875,131 @@ def test_dereference_schema_fail_invalid_max_ref_depth(bad_depth: int, exc_cls:
"""Reject non-positive or boolean max_ref_depth values."""
with pytest.raises(exc_cls, match="max_ref_depth"):
dereference_schema({"type": "object"}, max_ref_depth=bad_depth)
+
+
+def _build_ref_chain_schema(hops: int) -> dict[str, Any]:
+ """Build a Draft 2020-12 schema with a linear $ref chain of `hops` hops."""
+ defs: dict[str, Any] = {f"D{i}": {"$ref": f"#/$defs/D{i + 1}"} for i in range(hops - 1)}
+ defs[f"D{hops - 1}"] = {"type": "string"}
+ return {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$ref": "#/$defs/D0",
+ "$defs": defs,
+ }
+
+
+def test_dereference_schema_pass_default_max_ref_depth_allows_fifty_hops() -> None:
+ """Default max_ref_depth allows a 50-hop $ref chain."""
+ assert dereference_schema(_build_ref_chain_schema(50)) == {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "type": "string",
+ }
+
+
+def test_dereference_schema_fail_default_max_ref_depth_rejects_fifty_one_hops() -> None:
+ """Default max_ref_depth rejects a 51-hop $ref chain."""
+ with pytest.raises(DereferencingError, match=r"exceeded max depth \(50\)"):
+ dereference_schema(_build_ref_chain_schema(51))
+
+
+@pytest.mark.parametrize(
+ ("schema", "expected"),
+ [
+ pytest.param(
+ {"$ref": "#/$defs/Mid", "title": "Code"},
+ {"type": "string", "minLength": 1, "title": "Code"},
+ id="chained-ref-with-siblings",
+ ),
+ pytest.param(
+ {"allOf": [{"type": "object", "properties": {"a": {"type": "integer"}}}]},
+ {"type": "object", "properties": {"a": {"type": "integer"}}},
+ id="allof-resolution",
+ ),
+ ],
+)
+def test_resolve_ref_and_all_of_pass_siblings_chaining_and_allof(
+ schema: dict[str, Any],
+ expected: dict[str, Any],
+) -> None:
+ """_resolve_ref_and_all_of resolves chained $ref siblings and allOf and cleans up active state."""
+ root = {
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
+ "$defs": {
+ "Leaf": {"type": "string"},
+ "Mid": {"$ref": "#/$defs/Leaf", "minLength": 1},
+ },
+ }
+ ctx = _make_ctx(root)
+ assert _resolve_ref_and_all_of(schema, ctx) == expected
+ assert ctx.active_refs == set()
+ assert ctx.active_target_ids == set()
+
+
+@pytest.mark.parametrize(
+ ("ref_schema", "expected_warning"),
+ [
+ pytest.param(
+ {"$ref": "#/definitions/Leaf", "title": "T", "description": "D"},
+ "Ignoring keyword(s) ['description', 'title'] declared alongside $ref '#/definitions/Leaf'",
+ id="with-siblings-logs-warning",
+ ),
+ pytest.param(
+ {"$ref": "#/definitions/Leaf"},
+ None,
+ id="without-siblings-no-warning",
+ ),
+ ],
+)
+def test_resolve_ref_and_all_of_pass_draft7_sibling_warning(
+ caplog: pytest.LogCaptureFixture,
+ ref_schema: dict[str, Any],
+ expected_warning: str | None,
+) -> None:
+ """_resolve_ref_and_all_of logs ignored Draft-07 $ref siblings only when siblings are present."""
+ draft7_root = {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "definitions": {"Leaf": {"type": "string"}},
+ }
+ ctx = _make_ctx(draft7_root, spec=DRAFT7)
+ with caplog.at_level("WARNING"):
+ result = _resolve_ref_and_all_of(ref_schema, ctx)
+ assert result == {"type": "string"}
+ if expected_warning is None:
+ assert "Ignoring keyword" not in caplog.text
+ else:
+ assert expected_warning in caplog.text
+
+
+def test_merge_all_of_fail_non_list_all_of() -> None:
+ """_merge_all_of raises DereferencingError when allOf is not a list."""
+ ctx = _make_ctx({})
+ with pytest.raises(DereferencingError, match=r"'allOf' must be a list of schemas, got \{'type': 'string'\}\."):
+ _merge_all_of({"allOf": {"type": "string"}}, ctx)
+
+
+def test_merge_schema_pair_pass_concatenates_all_of_lists() -> None:
+ """_merge_schema_pair concatenates allOf lists from both schemas."""
+ merged = _merge_schema_pair({"allOf": [{"type": "string"}]}, {"allOf": [{"minLength": 2}]})
+ assert merged == {"allOf": [{"type": "string"}, {"minLength": 2}]}
+
+
+def test_dereference_schema_pass_dependencies_and_non_schema_single_keyword() -> None:
+ """dereference_schema resolves dict and bool dependencies and preserves list dependencies and non-schema values."""
+ schema = {
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "type": "object",
+ "dependencies": {
+ "a": ["b"],
+ "c": {"$ref": "#/definitions/Req"},
+ "d": True,
+ },
+ "not": "non-schema-passthrough",
+ "definitions": {"Req": {"required": ["e"]}},
+ }
+ result = dereference_schema(schema)
+ assert result["dependencies"] == {
+ "a": ["b"],
+ "c": {"required": ["e"]},
+ "d": True,
+ }
+ assert result["not"] == "non-schema-passthrough"
diff --git a/tests/cdm_data_loaders/validation/jsonschema/test_metaschema.py b/tests/cdm_data_loaders/validation/jsonschema/test_metaschema.py
index 79abb39b..2e9893f9 100644
--- a/tests/cdm_data_loaders/validation/jsonschema/test_metaschema.py
+++ b/tests/cdm_data_loaders/validation/jsonschema/test_metaschema.py
@@ -3,7 +3,7 @@
import importlib.resources
import json
from pathlib import Path
-from typing import Any, Final, Never
+from typing import Any, Final, Never, Self
import httpx
import jsonschema.exceptions
@@ -486,3 +486,74 @@ def test_check_jsonschema_document_fail_invalid_schemas(
"""Reject missing, non-string, unsupported, or metaschema-invalid JSON Schema documents."""
with pytest.raises(exc_cls, match=match):
check_jsonschema_document(bad_schema)
+
+
+def test_check_jsonschema_document_fail_disallowed_custom_xsv() -> None:
+ """Reject CUSTOM_META_SCHEMA_URI in check_jsonschema_document when allow_custom_xsv is False."""
+ with pytest.raises(ValueError, match=r"Custom metaschema cannot declare itself"):
+ check_jsonschema_document({"$schema": CUSTOM_META_SCHEMA_URI}, allow_custom_xsv=False)
+
+
+def test_register_xsv_validator_pass_idempotent() -> None:
+ """register_xsv_validator is idempotent across repeated calls."""
+ with preserve_xsv_metaschema_state():
+ register_xsv_validator()
+ first_cls = jsonschema.validators.validator_for({"$schema": CUSTOM_META_SCHEMA_URI}, default=None) # type: ignore[arg-type]
+ register_xsv_validator()
+ second_cls = jsonschema.validators.validator_for({"$schema": CUSTOM_META_SCHEMA_URI}, default=None) # type: ignore[arg-type]
+ assert first_cls is not None
+ assert first_cls is second_cls
+
+
+def test_register_xsv_validator_pass_adds_missing_id(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ """register_xsv_validator injects CUSTOM_META_SCHEMA_URI as $id when missing."""
+ no_id_dir = tmp_path / "xsv_config" / "0.1.0"
+ no_id_dir.mkdir(parents=True)
+ (no_id_dir / "schema.json").write_text(
+ json.dumps({"$schema": DRAFT_2020_12_SCHEMA_URI, "type": "object"}),
+ encoding="utf-8",
+ )
+ with preserve_xsv_metaschema_state():
+ monkeypatch.setattr(importlib.resources, "files", lambda _pkg: tmp_path)
+ register_xsv_validator()
+ cls = jsonschema.validators.validator_for({"$schema": CUSTOM_META_SCHEMA_URI}, default=None) # type: ignore[arg-type]
+ assert cls is not None
+ assert cls.META_SCHEMA["$id"] == CUSTOM_META_SCHEMA_URI
+
+
+def test_load_xsv_meta_schema_fail_forwards_remote_url_and_rejects_self_reference(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ """load_xsv_meta_schema forwards remote_url on fallback and rejects schemas declaring remote_url as $schema."""
+ custom_url = "https://example.org/custom-xsv/schema.json"
+ captured_urls: list[str] = []
+
+ class _FakeResponse:
+ def raise_for_status(self) -> None:
+ return None
+
+ def json(self) -> dict[str, Any]:
+ return {"$schema": custom_url, "type": "object"}
+
+ class _FakeClient:
+ def __init__(self, **_kwargs: object) -> None:
+ pass
+
+ def __enter__(self) -> Self:
+ return self
+
+ def __exit__(self, *_args: object) -> None:
+ return None
+
+ def get(self, url: str) -> _FakeResponse:
+ captured_urls.append(url)
+ return _FakeResponse()
+
+ monkeypatch.setattr(httpx, "Client", _FakeClient)
+ with pytest.raises(ValueError, match=r"Custom metaschema cannot declare itself"):
+ load_xsv_meta_schema(local_schema_path=tmp_path / "missing.json", remote_url=custom_url)
+ assert captured_urls == [custom_url]
diff --git a/tests/cdm_data_loaders/validation/xsd/test_particles.py b/tests/cdm_data_loaders/validation/xsd/test_particles.py
index 45e88d22..1a48cde9 100644
--- a/tests/cdm_data_loaders/validation/xsd/test_particles.py
+++ b/tests/cdm_data_loaders/validation/xsd/test_particles.py
@@ -170,3 +170,190 @@ def test_validate_group_particles_rejects_unsupported_constructs(body_xml: str,
assert any(is_any_element(item) for item in group.content)
with pytest.raises(ValueError, match=expected_match):
validate_group_particles(schema, group)
+
+
+GROUP_REF_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+"""
+
+NESTED_GROUPS_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+GROUP_OCCURS_EDGE_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+
+class CustomFallbackError(Exception):
+ """Test-local fallback exception class."""
+
+
+class CustomCodeError(Exception):
+ """Test-local code-mapped exception class."""
+
+
+@pytest.mark.parametrize(
+ ("xsd_text", "element_tag", "error_by_code", "expected_exc", "expected_match"),
+ [
+ pytest.param(
+ GROUP_REF_XSD,
+ "root",
+ {},
+ CustomFallbackError,
+ "xs:any wildcard element is not supported",
+ id="group-ref-fallback",
+ ),
+ pytest.param(
+ GROUP_REF_XSD,
+ "root",
+ {"E4": CustomCodeError},
+ CustomCodeError,
+ "xs:any wildcard element is not supported",
+ id="group-ref-mapped-e4",
+ ),
+ pytest.param(
+ NESTED_GROUPS_XSD,
+ "nestedAbs",
+ {},
+ CustomFallbackError,
+ "abstract element 'absEl' is not supported",
+ id="nested-abstract-element-fallback",
+ ),
+ pytest.param(
+ NESTED_GROUPS_XSD,
+ "nestedAbs",
+ {"E6": CustomCodeError},
+ CustomCodeError,
+ "abstract element 'absEl' is not supported",
+ id="nested-abstract-element-mapped-e6",
+ ),
+ pytest.param(
+ NESTED_GROUPS_XSD,
+ "nestedSub",
+ {},
+ CustomFallbackError,
+ "substitutionGroup on element 'headEl' is not supported",
+ id="nested-substitution-group-fallback",
+ ),
+ pytest.param(
+ NESTED_GROUPS_XSD,
+ "nestedSub",
+ {"E7": CustomCodeError},
+ CustomCodeError,
+ "substitutionGroup on element 'headEl' is not supported",
+ id="nested-substitution-group-mapped-e7",
+ ),
+ pytest.param(
+ NESTED_GROUPS_XSD,
+ "nestedAbsType",
+ {},
+ CustomFallbackError,
+ "abstract type 'AbsType' is not supported",
+ id="nested-abstract-type-fallback",
+ ),
+ pytest.param(
+ NESTED_GROUPS_XSD,
+ "nestedAbsType",
+ {"E6": CustomCodeError},
+ CustomCodeError,
+ "abstract type 'AbsType' is not supported",
+ id="nested-abstract-type-mapped-e6",
+ ),
+ ],
+)
+def test_validate_group_particles_fail_group_ref_and_nested_error_forwarding(
+ xsd_text: str,
+ element_tag: str,
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+ expected_match: str,
+) -> None:
+ """validate_group_particles resolves xs:group refs, recurses into nested groups, and forwards error classes."""
+ schema = xmlschema.XMLSchema(xsd_text)
+ group = schema.elements[element_tag].type.model_group # type: ignore[union-attr]
+ assert isinstance(group, XsdGroup)
+ with pytest.raises(expected_exc, match=expected_match):
+ validate_group_particles(schema, group, error_type=CustomFallbackError, error_by_code=error_by_code)
+
+
+@pytest.mark.parametrize(
+ ("element_tag", "expected_min"),
+ [
+ pytest.param("optGroup", 0, id="min-occurs-zero-group"),
+ pytest.param("emptyGroup", 0, id="empty-group"),
+ ],
+)
+def test_group_min_occurs_by_name_pass_zero_and_empty(element_tag: str, expected_min: int) -> None:
+ """group_min_occurs_by_name returns 0 for minOccurs=0 and empty model groups."""
+ schema = xmlschema.XMLSchema(GROUP_OCCURS_EDGE_XSD)
+ group = schema.elements[element_tag].type.model_group # type: ignore[union-attr]
+ assert isinstance(group, XsdGroup)
+ assert group_min_occurs_by_name(group, "a") == expected_min
+
+
+@pytest.mark.parametrize(
+ ("element_tag", "expected_max"),
+ [
+ pytest.param("zeroMaxGroup", 0, id="max-occurs-zero-group"),
+ pytest.param("emptyGroup", 0, id="empty-group"),
+ pytest.param("unboundedGroup", None, id="unbounded-group"),
+ ],
+)
+def test_group_max_occurs_by_name_pass_zero_empty_and_unbounded(
+ element_tag: str,
+ expected_max: int | None,
+) -> None:
+ """group_max_occurs_by_name handles maxOccurs=0, empty groups, and unbounded groups."""
+ schema = xmlschema.XMLSchema(GROUP_OCCURS_EDGE_XSD)
+ group = schema.elements[element_tag].type.model_group # type: ignore[union-attr]
+ assert isinstance(group, XsdGroup)
+ assert group_max_occurs_by_name(group, "a") == expected_max
diff --git a/tests/cdm_data_loaders/validation/xsd/test_scalars.py b/tests/cdm_data_loaders/validation/xsd/test_scalars.py
index aa8c818c..9f9c599c 100644
--- a/tests/cdm_data_loaders/validation/xsd/test_scalars.py
+++ b/tests/cdm_data_loaders/validation/xsd/test_scalars.py
@@ -181,3 +181,55 @@ def test_extract_doc_comment_pass_ref_fallback_and_missing(
child = children[child_name]
fallback_type = None if child_name == "globalRef" else child.type
assert extract_doc_comment(child, fallback_type) == expected_doc
+
+
+def test_extract_doc_comment_pass_whitespace_only_without_ref() -> None:
+ """extract_doc_comment returns None for whitespace-only documentation on a component without ref."""
+ no_ref = SimpleNamespace(annotation=SimpleNamespace(documentation=[SimpleNamespace(text=" \n")]))
+ assert extract_doc_comment(no_ref) is None
+
+
+def test_builtin_xsd_name_pass_missing_attributes() -> None:
+ """builtin_xsd_name falls back to xs:string when name and base_type attributes are absent."""
+ no_name = SimpleNamespace()
+ assert builtin_xsd_name(no_name) == "xs:string" # type: ignore[arg-type]
+
+
+def _return_false() -> bool:
+ """Return False for stub XSD type predicates.
+
+ :return: Always ``False``.
+ :rtype: bool
+ """
+ return False
+
+
+def _return_true() -> bool:
+ """Return True for stub XSD type predicates.
+
+ :return: Always ``True``.
+ :rtype: bool
+ """
+ return True
+
+
+def test_resolve_xsd_scalar_kind_pass_unknown_xsd_name() -> None:
+ """resolve_xsd_scalar_kind falls back to string for an unknown XSD scalar name."""
+ unknown_type = SimpleNamespace(
+ is_union=_return_false,
+ name="{http://www.w3.org/2001/XMLSchema}unknownCustom",
+ local_name="unknownCustom",
+ base_type=None,
+ )
+ assert resolve_xsd_scalar_kind(unknown_type) == ("string", "xs:unknownCustom") # type: ignore[arg-type]
+
+
+def test_iter_simple_builtin_names_pass_list_without_item_type() -> None:
+ """iter_simple_builtin_names falls back to xs:string for a list type missing item_type."""
+ broken_list = SimpleNamespace(
+ name=None,
+ base_type=None,
+ is_list=_return_true,
+ is_union=_return_false,
+ )
+ assert iter_simple_builtin_names(broken_list) == ["xs:string"] # type: ignore[arg-type]
diff --git a/tests/cdm_data_loaders/validation/xsd/test_traversal.py b/tests/cdm_data_loaders/validation/xsd/test_traversal.py
index 02c7c11e..de5983f1 100644
--- a/tests/cdm_data_loaders/validation/xsd/test_traversal.py
+++ b/tests/cdm_data_loaders/validation/xsd/test_traversal.py
@@ -332,3 +332,801 @@ def test_prepare_target_traversal_fail_invalid_max_recursion_depth(bad_depth: in
schema = xmlschema.XMLSchema(HIERARCHY_XSD)
with pytest.raises(exc_cls, match="max_recursion_depth"):
prepare_target_traversal(schema, "record", max_recursion_depth=bad_depth)
+
+
+CAN_REACH_DESCENDANT_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+CAN_REACH_VISITED_CYCLE_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+FORBIDDEN_ATTR_RECURSION_XSD: Final[str] = """
+
+
+
+
+"""
+
+FORBIDDEN_SIMPLE_CONTENT_RECURSION_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+"""
+
+NAMESPACED_RECURSION_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+DEEP_ELEMENT_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+AMBIGUOUS_ELEMENT_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+ABSTRACT_ELEMENT_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+NO_GLOBAL_ELEMENTS_XSD: Final[str] = """
+
+"""
+
+MUTUAL_CYCLE_XSD: Final[str] = """
+
+
+
+
+
+
+"""
+
+SELF_REF_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+"""
+
+ZERO_OCCURS_BEFORE_REF_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+"""
+
+AMBIGUOUS_ROOTS_XSD: Final[str] = """
+
+
+
+
+
+
+"""
+
+WILDCARD_ANY_XSD: Final[str] = """
+
+
+
+
+
+
+
+"""
+
+MIXED_NON_STRUCT_PARENTS_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+TARGET_STRUCT_EDGE_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+COLLISION_XSD: Final[str] = """
+
+
+
+
+
+
+"""
+
+SHARED_BAD_DAG_XSD: Final[str] = """
+
+
+
+
+
+
+
+
+
+
+
+"""
+
+LEAF_ONLY_XSD: Final[str] = """
+
+"""
+
+
+class CustomFallbackError(Exception):
+ """Test-local fallback exception class."""
+
+
+class CustomCodeError(Exception):
+ """Test-local code-mapped exception class."""
+
+
+@pytest.mark.parametrize(
+ ("error_by_code", "expected_exc"),
+ [
+ pytest.param({}, CustomFallbackError, id="fallback-error-type"),
+ pytest.param({"E1": CustomCodeError}, CustomCodeError, id="mapped-error-code"),
+ ],
+)
+def test_traversal_context_raise_err_fail_fallback_and_code_map(
+ hierarchy_schema: xmlschema.XMLSchema,
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+) -> None:
+ """TraversalContext.raise_err uses error_by_code when matched and error_type otherwise."""
+ ctx = TraversalContext(
+ schema=hierarchy_schema,
+ type_counts={},
+ max_recursion_depth=2,
+ error_type=CustomFallbackError,
+ error_by_code=error_by_code,
+ )
+ with pytest.raises(expected_exc, match="boom"):
+ ctx.raise_err("E1", "boom")
+
+
+@pytest.mark.parametrize(
+ ("xsd_text", "expected"),
+ [
+ pytest.param(
+ """
+
+
+
+ """,
+ True,
+ id="prohibited-attr-only",
+ ),
+ pytest.param(
+ """
+
+
+
+ """,
+ True,
+ id="zero-max-occurs-child-only",
+ ),
+ pytest.param(
+ """
+
+
+
+ """,
+ False,
+ id="optional-attr",
+ ),
+ pytest.param(
+ """
+
+
+
+ """,
+ False,
+ id="simple-content",
+ ),
+ ],
+)
+def test_is_empty_marker_complex_type_pass_edge_cases(xsd_text: str, expected: bool) -> None:
+ """is_empty_marker_complex_type handles prohibited attrs, maxOccurs=0, and simpleContent."""
+ schema = xmlschema.XMLSchema(xsd_text)
+ xsd_type = schema.types["T"]
+ assert isinstance(xsd_type, XsdComplexType)
+ assert is_empty_marker_complex_type(xsd_type) is expected
+
+
+def test_collect_attributes_pass_skips_prohibited() -> None:
+ """collect_attributes omits prohibited attributes and keeps optional ones."""
+ xsd_ok = """
+
+
+
+
+ """
+ schema = xmlschema.XMLSchema(xsd_ok)
+ xsd_type = schema.types["T"]
+ assert isinstance(xsd_type, XsdComplexType)
+ assert [a.local_name for a in collect_attributes(xsd_type)] == ["keep"]
+
+
+@pytest.mark.parametrize(
+ ("error_by_code", "expected_exc"),
+ [
+ pytest.param({}, CustomFallbackError, id="fallback-error"),
+ pytest.param({"E5": CustomCodeError}, CustomCodeError, id="mapped-e5-error"),
+ ],
+)
+def test_collect_attributes_fail_wildcard_attribute(
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+) -> None:
+ """collect_attributes raises E5 on xs:anyAttribute using error_by_code or error_type."""
+ xsd_bad = """
+
+
+
+ """
+ schema = xmlschema.XMLSchema(xsd_bad)
+ xsd_type = schema.types["T"]
+ assert isinstance(xsd_type, XsdComplexType)
+ with pytest.raises(expected_exc, match="xs:anyAttribute"):
+ collect_attributes(xsd_type, error_type=CustomFallbackError, error_by_code=error_by_code)
+
+
+@pytest.mark.parametrize(
+ ("xsd_text", "start_type_name", "target_type_name", "expected"),
+ [
+ pytest.param(
+ CAN_REACH_DESCENDANT_XSD,
+ "Wrapper",
+ "Target",
+ True,
+ id="reachable-through-cyclic-middle",
+ ),
+ pytest.param(
+ CAN_REACH_DESCENDANT_XSD,
+ "ZeroWrapper",
+ "Target",
+ False,
+ id="blocked-by-max-occurs-zero",
+ ),
+ pytest.param(
+ CAN_REACH_VISITED_CYCLE_XSD,
+ "Root",
+ "Target",
+ True,
+ id="continues-after-visited-cycle",
+ ),
+ ],
+)
+def test_can_reach_any_type_pass_descendants_zero_occurs_and_cycles(
+ xsd_text: str,
+ start_type_name: str,
+ target_type_name: str,
+ expected: bool,
+) -> None:
+ """can_reach_any_type descends into non-zero complex children and continues past visited cycles."""
+ schema = xmlschema.XMLSchema(xsd_text)
+ start_type = schema.types[start_type_name]
+ target_type = schema.types[target_type_name]
+ assert isinstance(start_type, XsdComplexType)
+ assert can_reach_any_type(start_type, {id(target_type): 1}) is expected
+
+
+@pytest.mark.parametrize(
+ ("xsd_text", "seed_type_count", "error_type", "error_by_code", "expected_exc", "expected_match"),
+ [
+ pytest.param(
+ FORBIDDEN_ATTR_RECURSION_XSD,
+ True,
+ CustomFallbackError,
+ {},
+ CustomFallbackError,
+ "@when",
+ id="date-attribute-fallback",
+ ),
+ pytest.param(
+ FORBIDDEN_SIMPLE_CONTENT_RECURSION_XSD,
+ True,
+ CustomFallbackError,
+ {"E9": CustomCodeError},
+ CustomCodeError,
+ "xs:dateTime",
+ id="simple-content-datetime-mapped-e9",
+ ),
+ pytest.param(
+ NAMESPACED_RECURSION_XSD,
+ False,
+ ValueError,
+ {},
+ ValueError,
+ "recursive type 'NodeType' contains",
+ id="namespaced-recursive-type-local-name",
+ ),
+ ],
+)
+def test_check_no_forbidden_types_under_recursion_fail_forbidden_members(
+ xsd_text: str,
+ seed_type_count: bool,
+ error_type: type[Exception],
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+ expected_match: str,
+) -> None:
+ """check_no_forbidden_types_under_recursion rejects date attributes, simpleContent extensions, and namespaced types."""
+ schema = xmlschema.XMLSchema(xsd_text)
+ node_type = schema.types["NodeType"]
+ type_counts = {id(node_type): 1} if seed_type_count else {}
+ ctx = TraversalContext(
+ schema=schema,
+ type_counts=type_counts,
+ max_recursion_depth=2,
+ error_type=error_type,
+ error_by_code=error_by_code,
+ )
+ with pytest.raises(expected_exc, match=expected_match):
+ check_no_forbidden_types_under_recursion(node_type, ctx)
+
+
+def test_check_no_forbidden_types_under_recursion_pass_mixed_type() -> None:
+ """check_no_forbidden_types_under_recursion skips model_group inspection for mixed complex types."""
+ schema = xmlschema.XMLSchema(NAMESPACED_RECURSION_XSD)
+ mixed_type = schema.types["MixedRec"]
+ ctx = TraversalContext(schema=schema, type_counts={}, max_recursion_depth=2)
+ check_no_forbidden_types_under_recursion(mixed_type, ctx)
+
+
+def test_resolve_target_element_pass_nested_clark_name() -> None:
+ """resolve_target_element finds a nested qualified element by Clark-notation tag."""
+ schema = xmlschema.XMLSchema(DEEP_ELEMENT_XSD)
+ found = resolve_target_element(schema, "{http://example.org/ns}item")
+ assert found.local_name == "item"
+
+
+@pytest.mark.parametrize(
+ ("xsd_text", "xml_tag", "error_by_code", "expected_exc", "expected_match"),
+ [
+ pytest.param(DEEP_ELEMENT_XSD, "missing", {}, CustomFallbackError, "not found", id="missing-tag-fallback"),
+ pytest.param(
+ DEEP_ELEMENT_XSD,
+ "missing",
+ {"E1": CustomCodeError},
+ CustomCodeError,
+ "not found",
+ id="missing-tag-mapped-e1",
+ ),
+ pytest.param(
+ AMBIGUOUS_ELEMENT_XSD,
+ "dup",
+ {},
+ CustomFallbackError,
+ "Ambiguous xml_tag 'dup'",
+ id="ambiguous-tag-fallback",
+ ),
+ pytest.param(
+ AMBIGUOUS_ELEMENT_XSD,
+ "dup",
+ {"E2": CustomCodeError},
+ CustomCodeError,
+ "Ambiguous xml_tag 'dup'",
+ id="ambiguous-tag-mapped-e2",
+ ),
+ pytest.param(
+ ABSTRACT_ELEMENT_XSD,
+ "absEl",
+ {},
+ CustomFallbackError,
+ "abstract element 'absEl'",
+ id="abstract-element-fallback",
+ ),
+ pytest.param(
+ ABSTRACT_ELEMENT_XSD,
+ "absTyped",
+ {"E6": CustomCodeError},
+ CustomCodeError,
+ "abstract type on 'absTyped'",
+ id="abstract-type-mapped-e6",
+ ),
+ ],
+)
+def test_resolve_target_element_fail_missing_ambiguous_and_abstract(
+ xsd_text: str,
+ xml_tag: str,
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+ expected_match: str,
+) -> None:
+ """resolve_target_element forwards E1, E2, and E6 errors for missing, ambiguous, and abstract declarations."""
+ schema = xmlschema.XMLSchema(xsd_text)
+ with pytest.raises(expected_exc, match=expected_match):
+ resolve_target_element(schema, xml_tag, error_type=CustomFallbackError, error_by_code=error_by_code)
+
+
+@pytest.mark.parametrize(
+ ("error_by_code", "expected_exc"),
+ [
+ pytest.param({}, CustomFallbackError, id="fallback-error"),
+ pytest.param({"E1": CustomCodeError}, CustomCodeError, id="mapped-e1-error"),
+ ],
+)
+def test_resolve_root_elements_fail_no_global_elements(
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+) -> None:
+ """resolve_root_elements raises E1 when a schema declares no global elements."""
+ schema = xmlschema.XMLSchema(NO_GLOBAL_ELEMENTS_XSD)
+ with pytest.raises(expected_exc, match="No global elements"):
+ resolve_root_elements(schema, error_type=CustomFallbackError, error_by_code=error_by_code)
+
+
+@pytest.mark.parametrize(
+ ("xsd_text", "expected_names"),
+ [
+ pytest.param(MUTUAL_CYCLE_XSD, ["a", "b"], id="mutual-cycle"),
+ pytest.param(SELF_REF_XSD, ["node", "leaf"], id="self-ref-and-zero-occurs"),
+ pytest.param(ZERO_OCCURS_BEFORE_REF_XSD, ["skip_target", "root"], id="zero-occurs-before-active-ref"),
+ ],
+)
+def test_resolve_root_elements_pass_cycles_and_zero_occurs_refs(
+ xsd_text: str,
+ expected_names: list[str],
+) -> None:
+ """resolve_root_elements handles mutual cycles, self-references, and maxOccurs=0 refs."""
+ schema = xmlschema.XMLSchema(xsd_text)
+ assert [e.local_name for e in resolve_root_elements(schema)] == expected_names
+
+
+@pytest.mark.parametrize(
+ ("xsd_text", "level", "expected_names"),
+ [
+ pytest.param(HIERARCHY_XSD, 0, ["catalog"], id="level-0-root"),
+ pytest.param(
+ MIXED_NON_STRUCT_PARENTS_XSD,
+ 1,
+ ["k"],
+ id="skips-simple-mixed-and-duplicate-parents",
+ ),
+ ],
+)
+def test_resolve_elements_at_level_pass_level_zero_and_skips_non_struct_parents(
+ xsd_text: str,
+ level: int,
+ expected_names: list[str],
+) -> None:
+ """resolve_elements_at_level returns root elements at level 0 and skips simple, mixed, and duplicate parents."""
+ schema = xmlschema.XMLSchema(xsd_text)
+ assert list(resolve_elements_at_level(schema, level)) == expected_names
+
+
+@pytest.mark.parametrize(
+ ("xsd_text", "level", "root_tag", "error_by_code", "expected_exc", "expected_match"),
+ [
+ pytest.param(
+ HIERARCHY_XSD,
+ 5,
+ None,
+ {},
+ CustomFallbackError,
+ "No XML elements found at level 5",
+ id="empty-deep-level-fallback",
+ ),
+ pytest.param(
+ HIERARCHY_XSD,
+ 5,
+ None,
+ {"E1": CustomCodeError},
+ CustomCodeError,
+ "No XML elements found at level 5",
+ id="empty-deep-level-mapped-e1",
+ ),
+ pytest.param(
+ AMBIGUOUS_ROOTS_XSD,
+ 0,
+ "missing",
+ {},
+ CustomFallbackError,
+ "not found",
+ id="missing-root-tag-fallback",
+ ),
+ pytest.param(
+ AMBIGUOUS_ROOTS_XSD,
+ 0,
+ "missing",
+ {"E1": CustomCodeError},
+ CustomCodeError,
+ "not found",
+ id="missing-root-tag-mapped-e1",
+ ),
+ pytest.param(
+ AMBIGUOUS_ROOTS_XSD,
+ 1,
+ None,
+ {},
+ CustomFallbackError,
+ "Ambiguous xml_tag 'dup'",
+ id="ambiguous-child-fallback",
+ ),
+ pytest.param(
+ AMBIGUOUS_ROOTS_XSD,
+ 1,
+ None,
+ {"E2": CustomCodeError},
+ CustomCodeError,
+ "Ambiguous xml_tag 'dup'",
+ id="ambiguous-child-mapped-e2",
+ ),
+ pytest.param(
+ WILDCARD_ANY_XSD,
+ 1,
+ None,
+ {},
+ CustomFallbackError,
+ "xs:any",
+ id="wildcard-any-fallback",
+ ),
+ pytest.param(
+ WILDCARD_ANY_XSD,
+ 1,
+ None,
+ {"E4": CustomCodeError},
+ CustomCodeError,
+ "xs:any",
+ id="wildcard-any-mapped-e4",
+ ),
+ ],
+)
+def test_resolve_elements_at_level_fail_error_forwarding(
+ xsd_text: str,
+ level: int,
+ root_tag: str | None,
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+ expected_match: str,
+) -> None:
+ """resolve_elements_at_level forwards E1, E2, and E4 errors via error_by_code or error_type."""
+ schema = xmlschema.XMLSchema(xsd_text)
+ with pytest.raises(expected_exc, match=expected_match):
+ resolve_elements_at_level(
+ schema,
+ level,
+ root_tag=root_tag,
+ error_type=CustomFallbackError,
+ error_by_code=error_by_code,
+ )
+
+
+@pytest.mark.parametrize(
+ ("tag", "error_by_code", "expected_exc"),
+ [
+ pytest.param("empty", {}, CustomFallbackError, id="empty-fallback"),
+ pytest.param("empty", {"E3": CustomCodeError}, CustomCodeError, id="empty-mapped-e3"),
+ pytest.param("plainSc", {}, CustomFallbackError, id="attrless-simple-content-fallback"),
+ pytest.param("plainSc", {"E3": CustomCodeError}, CustomCodeError, id="attrless-simple-content-mapped-e3"),
+ ],
+)
+def test_validate_target_struct_element_fail_empty_and_attrless_simple_content(
+ tag: str,
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+) -> None:
+ """validate_target_struct_element rejects empty and attribute-less simpleContent target elements."""
+ schema = xmlschema.XMLSchema(TARGET_STRUCT_EDGE_XSD)
+ _, elem_type, ctx = prepare_target_traversal(
+ schema,
+ tag,
+ max_recursion_depth=2,
+ error_type=CustomFallbackError,
+ error_by_code=error_by_code,
+ )
+ with pytest.raises(expected_exc):
+ validate_target_struct_element(elem_type, tag, ctx)
+
+
+def test_validate_target_struct_element_pass_simple_content_with_attributes() -> None:
+ """validate_target_struct_element accepts simpleContent target elements that declare attributes."""
+ schema = xmlschema.XMLSchema(TARGET_STRUCT_EDGE_XSD)
+ _, elem_type, ctx = prepare_target_traversal(
+ schema,
+ "attrSc",
+ max_recursion_depth=2,
+ error_type=CustomFallbackError,
+ )
+ validate_target_struct_element(elem_type, "attrSc", ctx)
+
+
+@pytest.mark.parametrize(
+ ("xsd_text", "xml_tag", "error_by_code", "expected_exc", "expected_match"),
+ [
+ pytest.param(
+ COLLISION_XSD,
+ "rec",
+ {"E8": CustomCodeError},
+ CustomCodeError,
+ r"Field name collision.*'_id'",
+ id="direct-collision-mapped-e8",
+ ),
+ pytest.param(
+ SHARED_BAD_DAG_XSD,
+ "root",
+ {},
+ CustomFallbackError,
+ "Field name collision",
+ id="nested-collision-fallback",
+ ),
+ ],
+)
+def test_validate_complex_subtree_fail_field_name_collision(
+ xsd_text: str,
+ xml_tag: str,
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+ expected_match: str,
+) -> None:
+ """validate_complex_subtree detects E8 field-name collisions and forwards error classes."""
+ schema = xmlschema.XMLSchema(xsd_text)
+ _, complex_type, ctx = prepare_target_traversal(
+ schema,
+ xml_tag,
+ max_recursion_depth=2,
+ error_type=CustomFallbackError,
+ error_by_code=error_by_code,
+ )
+ with pytest.raises(expected_exc, match=expected_match):
+ validate_complex_subtree(complex_type, ctx)
+
+
+def test_validate_complex_subtree_pass_skips_visited_type() -> None:
+ """validate_complex_subtree skips complex types already present in the visited set."""
+ schema = xmlschema.XMLSchema(SHARED_BAD_DAG_XSD)
+ _, root_type, root_ctx = prepare_target_traversal(
+ schema,
+ "root",
+ max_recursion_depth=2,
+ error_type=CustomFallbackError,
+ )
+ visited = {id(schema.types["SharedBad"])}
+ validate_complex_subtree(root_type, root_ctx, visited=visited)
+
+
+@pytest.mark.parametrize(
+ ("xml_tag", "error_by_code", "expected_exc", "expected_match"),
+ [
+ pytest.param("missing", {}, CustomFallbackError, "not found", id="missing-tag-fallback"),
+ pytest.param(
+ "leaf",
+ {"E3": CustomCodeError},
+ CustomCodeError,
+ "must have a complex type",
+ id="simple-type-mapped-e3",
+ ),
+ ],
+)
+def test_prepare_target_traversal_fail_forwards_error_classes(
+ xml_tag: str,
+ error_by_code: dict[str, type[Exception]],
+ expected_exc: type[Exception],
+ expected_match: str,
+) -> None:
+ """prepare_target_traversal forwards error_type and error_by_code to element and type resolution."""
+ schema = xmlschema.XMLSchema(LEAF_ONLY_XSD)
+ with pytest.raises(expected_exc, match=expected_match):
+ prepare_target_traversal(
+ schema,
+ xml_tag,
+ max_recursion_depth=2,
+ error_type=CustomFallbackError,
+ error_by_code=error_by_code,
+ )
diff --git a/uv.lock b/uv.lock
index 818f1a8e..0bac33b5 100644
--- a/uv.lock
+++ b/uv.lock
@@ -34,6 +34,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/7f/bc/3bb8965204ba51bbf4d021bc89d1357cd2179a17ef71b5fc429e21b511a2/aiobotocore-3.10.0-py3-none-any.whl", hash = "sha256:de0ccbcb82b6b0009f1b6667f48442e9d0432f9b8bfe6172fa51777cbea89d03", size = 102759, upload-time = "2026-10-23T10:44:49.109Z" },
]
+[[package]]
+name = "aiofile"
+version = "3.12.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "caio" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/14/31/edb06aabd8f8f0b56d659f30800795f40b93cba96be946ce179f6931e3a5/aiofile-3.12.3.tar.gz", hash = "sha256:caa6aa746b5e47e2165f7abd741b6415e49cf4d44fddc0f61844612cc3924d41", size = 21600, upload-time = "2026-08-04T22:59:27.171Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/4e/79/6e45e778c4c3cab39e0937b007b720c15f76c50c6453d153282d0fcc3588/aiofile-3.12.3-py3-none-any.whl", hash = "sha256:5c1bcc9e929c50834608e8cc1a4cc1d7503eb60c15a535b779fd39e2f372c017", size = 22122, upload-time = "2026-08-04T22:59:25.838Z" },
+]
+
[[package]]
name = "aiohappyeyeballs"
version = "2.7.1"
@@ -282,6 +294,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" },
]
+[[package]]
+name = "authlib"
+version = "1.9.9"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cryptography" },
+ { name = "joserfc" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/73/69/0215a1818edb1fb6b9dd59b8a3870601f11603b6db1f830ff3f3ea62612f/authlib-1.9.9.tar.gz", hash = "sha256:cca3dffa51c8d80d6df19aee2279e47219dc996f5a527c06ac2e39d0757c99ef", size = 184173, upload-time = "2026-11-17T20:35:21.058Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ba/a5/e79946bc081bfd2e8c0c6125b5a6ae90f8d1541bb224cd7f16ccbb8e71d8/authlib-1.9.9-py2.py3-none-any.whl", hash = "sha256:0199649865fefb4304fc57683eaebb15614523073e41e09d98c72f3268ce5cbc", size = 267584, upload-time = "2026-11-17T20:35:19.249Z" },
+]
+
[[package]]
name = "aws-xray-sdk"
version = "2.16.0"
@@ -366,6 +391,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/86/3c/7297d87eed9254f6b4823c05b37aa07ec2a99bc5f195760dc574e925eecf/backports_zstd-1.8.0-cp313-cp313-win_arm64.whl", hash = "sha256:0600e166cb00739a26de74ee1696221a53a4d5dc1f96a0bdeb6b307c1626c15c", size = 322066, upload-time = "2026-10-10T16:36:04.932Z" },
]
+[[package]]
+name = "beartype"
+version = "0.23.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/79/34/c0cfdaee9c796c8381439fed0fcab62810753b8b0adb1ec88fa7f41eaa5d/beartype-0.23.1.tar.gz", hash = "sha256:8b805f246b32931c74f2b321ea0939fe5bb2ff43c12a3eff19cc4c854b832e1d", size = 1993714, upload-time = "2026-10-11T05:36:55.251Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/1c/cb/2f6aa982e2860c1156830db3da7f823b71784ed7d533ca289ea9b6c6347a/beartype-0.23.1-py3-none-any.whl", hash = "sha256:4461b4dc57e3fdd6c8a8464b22cf118002eea107c33e9fef651c69068ab3cce3", size = 1582101, upload-time = "2026-10-11T05:36:52.431Z" },
+]
+
[[package]]
name = "bioregistry"
version = "0.15.12"
@@ -480,6 +514,41 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a6/bc/2e35dcb4bfb37a5dd98b8b323158d3103cf263f87164ed03702f6fc5483f/cachetools-8.0.0-py3-none-any.whl", hash = "sha256:5af562e5078f0086771a1e3832a9c3dbf5d336b6e2794a2c12a64680a4c9087d", size = 18610, upload-time = "2026-11-07T07:30:30.831Z" },
]
+[[package]]
+name = "caio"
+version = "0.12.10"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/66/66/a36d1367bd5ef70b6b97b2fb502f682106555e9e49e229f35c0a49d154d5/caio-0.12.10.tar.gz", hash = "sha256:5cecc44c04d5c6291d98b61bdba999f49a7e58dff2d782b4bb0642dc6f7ecf58", size = 84118, upload-time = "2026-10-22T21:35:06.294Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/84/3e/98ddc781ae6d76407b74f33136f838e444df41a1a43d80373da01da47bbb/caio-0.12.10-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:6b8e0724415ae86889df59af2d44fd6a531851ebc62c9cfdb1cc8235c929524d", size = 49054, upload-time = "2026-10-22T21:34:33.252Z" },
+ { url = "https://files.pythonhosted.org/packages/8b/d6/9851c6b24674ef72054c7b05569bd444f1358955c19ba5b5f7e15fa64be6/caio-0.12.10-cp313-cp313-manylinux_2_34_aarch64.whl", hash = "sha256:4e72602d4c348d105c9e46dd300b85df3fd3e857e2068370847f5e29879181b5", size = 164861, upload-time = "2026-10-22T21:34:34.472Z" },
+ { url = "https://files.pythonhosted.org/packages/85/9a/1f6eac05eb23284c834c08c17cc5f39fafe3754f1d2df94bcf5a2cb93179/caio-0.12.10-cp313-cp313-manylinux_2_34_x86_64.whl", hash = "sha256:1d8b6e9275d8f9608b8ecb4b97189e7f1d0b8ed948fb552eb06518d6342a1e53", size = 162592, upload-time = "2026-10-22T21:34:35.566Z" },
+ { url = "https://files.pythonhosted.org/packages/1a/be/4bb0538aa4c953c4681c7b309a9cb452eb91640e6b62f250b464f91dfad4/caio-0.12.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:cff7a239334b36c4dc82a0ab69aa3244188e5c9e2c30a242d81e4502520f2b1b", size = 162324, upload-time = "2026-10-22T21:34:36.977Z" },
+ { url = "https://files.pythonhosted.org/packages/0a/9a/290190821dbc29fccb0ad96e51a7f573394df621da547cb13701de685805/caio-0.12.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4a9645aeed1cdeeff34ca78442e6b274c0f801876a6bf0991724d92b7d639655", size = 162171, upload-time = "2026-10-22T21:34:38.173Z" },
+ { url = "https://files.pythonhosted.org/packages/39/9d/2a2177a43901f69ef120fb0dfe4a7dc6938e57a086da3ecdeacc12a05635/caio-0.12.10-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:edda39ac24b4cec74486a06afe7499fc9ddb2c3d5cb98494b2c9fdf95ed7aecd", size = 42798, upload-time = "2026-10-22T21:34:39.444Z" },
+ { url = "https://files.pythonhosted.org/packages/b9/0a/5e404a09363c71dfb5f7c20234ee31dc9ad75c7072dc868b6cc5be6f9af3/caio-0.12.10-cp314-cp314-macosx_26_0_arm64.whl", hash = "sha256:3b9c6e6355267f2eccd23e1a17ec2f78dce02ab00b10276999ff7a4615512952", size = 42033, upload-time = "2026-10-22T21:34:40.448Z" },
+ { url = "https://files.pythonhosted.org/packages/06/94/41549be5a09a0441f1c1313d5eeb8b60b08034c89cd2c0b5cfeb6a9b9111/caio-0.12.10-cp314-cp314-manylinux_2_34_aarch64.whl", hash = "sha256:a2b897245d5e6593a39ea9a52c7b372efc41e5a746c46a2985bdf94622f273fe", size = 165369, upload-time = "2026-10-22T21:34:41.513Z" },
+ { url = "https://files.pythonhosted.org/packages/28/47/f272ce0eed9700749934823086434a94d9bea51555724188d8ee660a7fd4/caio-0.12.10-cp314-cp314-manylinux_2_34_x86_64.whl", hash = "sha256:40e84b8679f44bee5c2b7cc7093719e15616b6752560809896dc15a8667cd69f", size = 162620, upload-time = "2026-10-22T21:34:42.671Z" },
+ { url = "https://files.pythonhosted.org/packages/65/ae/644e7e72a81c4d8ce31dac9865b2a6b38fdc4c56a1418cc2d4d55591be20/caio-0.12.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:ccd4efddafd48018d30cd560f7c393d177fe01365ee0dcf0bc88528f220732ed", size = 162986, upload-time = "2026-10-22T21:34:43.801Z" },
+ { url = "https://files.pythonhosted.org/packages/63/2a/557a90ae71f8411a2cfa0238ad70bddb60178133b7eb41e3fe8e8c854235/caio-0.12.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:2138670dd6cab64bd29cfcbd6a7259c0c21646c4dc86a4ec12de907210a2aba9", size = 162258, upload-time = "2026-10-22T21:34:45.271Z" },
+ { url = "https://files.pythonhosted.org/packages/78/1a/1ffea8df50383d5bc468e94d2b45d1c458f965b80bb12ad28891d0a55266/caio-0.12.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:79a4e0000dd92b1e4e7842fca99fdaa5fd0259dd33a9892741e7126e74656c49", size = 42801, upload-time = "2026-10-22T21:34:46.485Z" },
+ { url = "https://files.pythonhosted.org/packages/f0/59/43da115afd82281312166711ceff5b7ca08d53f79a8955d2e22ac9968eca/caio-0.12.10-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:6fff1f609e54a2e007b6c5c0933f4782ecbf8b6f9144e0930e2dcd2e7440c637", size = 186884, upload-time = "2026-10-22T21:34:47.611Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/5d/708aed113a2ae829a265b5ce030bab52ff0e41339cc6047f29bb0b15dfdf/caio-0.12.10-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:925b938bcdf09d1c3df2b6ddd534004b616451ef1810e47f6a63257365376b25", size = 182799, upload-time = "2026-10-22T21:34:48.86Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/c3/8de2c1c89548902a389406560e67930333984194217d5bcef6129ff7f18a/caio-0.12.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:1ebcac2a14e045aa0a4f63824264a4a845c6b1ac9074d5d673c52ef564a1ada6", size = 185097, upload-time = "2026-10-22T21:34:50.035Z" },
+ { url = "https://files.pythonhosted.org/packages/94/43/98997e24fa52f1e6a2a808e446693702773830a863a036a5e727024e7aa0/caio-0.12.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:39e91aab7d0a5f0cac5930562d751e9938ceef1c7ae18477a5689a4e437eada1", size = 182593, upload-time = "2026-10-22T21:34:51.256Z" },
+ { url = "https://files.pythonhosted.org/packages/c3/17/78046176bc3572430d03bab1b9e82966bc5b9a28d27c41e6764a20144044/caio-0.12.10-cp315-cp315-macosx_26_0_arm64.whl", hash = "sha256:c0b52bd6fd006d2d07713025f0e1cad61ebd12342f86bfa47e2eff239fbdd01f", size = 42039, upload-time = "2026-10-22T21:34:52.646Z" },
+ { url = "https://files.pythonhosted.org/packages/ea/ed/14a90cfe02f14813674a1cc91ff2fad9b0d1bb98260be09b987f579baa28/caio-0.12.10-cp315-cp315-manylinux_2_34_aarch64.whl", hash = "sha256:b7a42ef7379e36e99cc6990aa62465919a3be3fb1f049a40e5aef29fbbd859e1", size = 166510, upload-time = "2026-10-22T21:34:53.703Z" },
+ { url = "https://files.pythonhosted.org/packages/c2/b1/9fbeafd6aae00cc0dae96d190cc4fe6e23d20ecd4f26e970878611761fdb/caio-0.12.10-cp315-cp315-manylinux_2_34_x86_64.whl", hash = "sha256:89fe0e3c849a07aba1f76012a50eaf4fb2eaed0398c01bd43912a1bd0f4ab5eb", size = 163501, upload-time = "2026-10-22T21:34:54.988Z" },
+ { url = "https://files.pythonhosted.org/packages/34/81/2342f17ab96a7b5d3666cc7dccea59728c4d421e7c954e2937cd817c3be5/caio-0.12.10-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:21b04f6722884e5fb19690743772875b34306e5ba68d9ecabf7454193bb5caab", size = 164139, upload-time = "2026-10-22T21:34:56.201Z" },
+ { url = "https://files.pythonhosted.org/packages/17/b3/e15cfdc902a13d885fa16481c7d01ad04de7b50e0195b9dd5e219ce07614/caio-0.12.10-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:a04ac9769fd8eddc1c0ca6308f0f6e668a86cf8d577770c11d85d0aeb584fcf4", size = 163137, upload-time = "2026-10-22T21:34:57.402Z" },
+ { url = "https://files.pythonhosted.org/packages/49/ad/7fda5be8557afa8833ea65ebb29feaeeb96cb2b3373fe3220fe1e6899260/caio-0.12.10-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:c8fc80a37f08bba4c2f108277b6eabba564f74963a1a6cab01e50f9ae4434681", size = 42800, upload-time = "2026-10-22T21:34:58.494Z" },
+ { url = "https://files.pythonhosted.org/packages/0d/e0/e2692eeb6841614f60ec099eac3c9b10a00dac74adf7321f442a2291ef8d/caio-0.12.10-cp315-cp315t-manylinux_2_34_aarch64.whl", hash = "sha256:3e3799d09b08401e83c30c671f6fa1e1227f6346a3b1336f24cb94c5a8a82b64", size = 187471, upload-time = "2026-10-22T21:34:59.585Z" },
+ { url = "https://files.pythonhosted.org/packages/cc/a8/2c3a0cd781732057ab6a8023ef0fe14017b29ef75259295df8f585e3dfba/caio-0.12.10-cp315-cp315t-manylinux_2_34_x86_64.whl", hash = "sha256:ea2e0ea832d36c1e50ab730ba304153f34eb7dc2300b1285fc7cdfae1f1f4075", size = 183526, upload-time = "2026-10-22T21:35:00.764Z" },
+ { url = "https://files.pythonhosted.org/packages/62/e2/9f32db16441875afaa2a8a27d9734c2c8421f259163d7d814528160fa6c8/caio-0.12.10-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:8d1071f8cb5d23c8aeccffe36c50a17d32257cddbf913b7e9b6fb524ba42d7a8", size = 185740, upload-time = "2026-10-22T21:35:02.059Z" },
+ { url = "https://files.pythonhosted.org/packages/0d/7d/17836b477a1e57a648b90f16a27ebd367d686a02e5b2a4e72b53ae5ad7fe/caio-0.12.10-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:2a54ddadb7db4eab50bfc8b34f36fcf6763e9007f86b92f7cd03f07b542e891e", size = 183396, upload-time = "2026-10-22T21:35:03.492Z" },
+ { url = "https://files.pythonhosted.org/packages/5c/c3/1c44005ec4d4c3f7c51b707308efe7836ee16c466f4ff2e2a6f2089eceee/caio-0.12.10-py3-none-any.whl", hash = "sha256:b34a5483b987d207348ebbf86921bc42f08abcde833b91e0caa8760c7daa2f03", size = 26166, upload-time = "2026-10-22T21:35:04.842Z" },
+]
+
[[package]]
name = "cdm-data-loaders"
version = "0.1.13"
@@ -540,6 +609,7 @@ models = [
]
mutants = [
{ name = "mutmut" },
+ { name = "mutmut-mcp" },
]
[package.metadata]
@@ -595,7 +665,10 @@ models = [
{ name = "genson", specifier = ">=1.4.0" },
{ name = "json2python-models", specifier = ">=0.3.1" },
]
-mutants = [{ name = "mutmut", specifier = ">=3.8.1" }]
+mutants = [
+ { name = "mutmut", specifier = ">=3.8.1" },
+ { name = "mutmut-mcp", git = "https://github.com/wdm0006/mutmut-mcp" },
+]
[[package]]
name = "cdm-task-service-client"
@@ -1169,6 +1242,21 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/34/00/5ecde1db0e2bfed9e4613459f07f553145387f20f03fdcb142f9f9c43ccf/curies-0.15.8-py3-none-any.whl", hash = "sha256:f46c58ffedebeecb4a9122eb0df4719907cd0d624be29888a9845f8867887aa2", size = 86104, upload-time = "2026-11-10T19:43:09.901Z" },
]
+[[package]]
+name = "cyclopts"
+version = "5.5.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "attrs" },
+ { name = "docstring-parser" },
+ { name = "rich" },
+ { name = "rich-rst" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/49/5d/cee0096dd8c5ad13ce516599c4823637d0107643666a03ddc43a9580391f/cyclopts-5.5.2.tar.gz", hash = "sha256:c0339b7ddd7b1c886f13bb8b68a6a65aa7144d3027f1d0e290be514b8fe1589b", size = 258336, upload-time = "2026-11-16T16:26:28.322Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7a/b2/be66b75ce0a3ccc7aa3601c9e7f9cd0fbad4301183fc7105c6b9faebeaeb/cyclopts-5.5.2-py3-none-any.whl", hash = "sha256:4fbe863ebace1c65dafc6520c82217915b103d27fbd1d8fcd168f2abcca3a12a", size = 301206, upload-time = "2026-11-16T16:26:26.615Z" },
+]
+
[[package]]
name = "datamodel-code-generator"
version = "0.93.0"
@@ -1322,6 +1410,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/75/23/529140fe1aab80fc6992f93a706deec709140a6397439139a054e1515c45/docker-7.2.0-py3-none-any.whl", hash = "sha256:a3f45fdeb9165e2d25d9a1d02ddf3bc70fb572cf5ebbf9b58558c22caf29b71f", size = 148775, upload-time = "2026-07-09T14:53:45.224Z" },
]
+[[package]]
+name = "docstring-parser"
+version = "0.18.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e0/4d/f332313098c1de1b2d2ff91cf2674415cc7cddab2ca1b01ae29774bd5fdf/docstring_parser-0.18.0.tar.gz", hash = "sha256:292510982205c12b1248696f44959db3cdd1740237a968ea1e2e7a900eeb2015", size = 29341, upload-time = "2026-04-14T04:09:19.867Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a7/5f/ed01f9a3cdffbd5a008556fc7b2a08ddb1cc6ace7effa7340604b1d16699/docstring_parser-0.18.0-py3-none-any.whl", hash = "sha256:b3fcbed555c47d8479be0796ef7e19c2670d428d72e96da63f3a40122860374b", size = 22484, upload-time = "2026-04-14T04:09:18.638Z" },
+]
+
[[package]]
name = "docutils"
version = "0.23"
@@ -1384,6 +1481,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c1/8b/5fe2cc11fee489817272089c4203e679c63b570a5aaeb18d852ae3cbba6a/et_xmlfile-2.0.0-py3-none-any.whl", hash = "sha256:7a91720bc756843502c3b7504c77b8fe44217c85c537d85037f0f536151b2caa", size = 18059, upload-time = "2024-10-25T17:25:39.051Z" },
]
+[[package]]
+name = "exceptiongroup"
+version = "1.3.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/85/ac/f882fe3234606340c8729372da0fb98c634b97a560c9d375d9afea21496b/exceptiongroup-1.3.2.tar.gz", hash = "sha256:ca82e69f0cb535fbde934885744b67669a3efddd50fe7bc00d3398839fcaddec", size = 34687, upload-time = "2026-11-19T22:10:16.815Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a0/7d/0fb19ccdf794c448141461180d5992c50402f69f8515a801bfac7ad65f45/exceptiongroup-1.3.2-py3-none-any.whl", hash = "sha256:41c6a70dfec22050d3d8b43386b059a4dc0bca3f3d40a40971cb206fff1785af", size = 16854, upload-time = "2026-11-19T22:10:15.746Z" },
+]
+
[[package]]
name = "executing"
version = "2.3.0"
@@ -1393,6 +1499,71 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/4d/dd/8bc67e7d5ffc1d88aa5af511189dfb76dc4e1e5b808fab186146ef2663e5/executing-2.3.0-py3-none-any.whl", hash = "sha256:736e859c9f8701f11fcf516856f26f562e04776387824b43a35a1dfe21c84122", size = 29393, upload-time = "2026-10-10T14:06:02.777Z" },
]
+[[package]]
+name = "fastmcp"
+version = "3.4.11"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "fastmcp-slim", extra = ["client", "server"] },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/f7/77/b5ef8688b0dd3e403b7671106533d62e93252d7d8f3c75f547e4686bba1e/fastmcp-3.4.11.tar.gz", hash = "sha256:36273b04eff98021da4601d4f86c64dfa10e61119e9ddd17ad80650465c65355", size = 28868316, upload-time = "2026-10-20T22:44:36.64Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d5/fb/e6b04a2f4dbce64994f3239267b6899411e8fd9693238c39ee99fe6f2b35/fastmcp-3.4.11-py3-none-any.whl", hash = "sha256:f50a7aa4cc2a5954d02fcc9672ff1d69daff73a02387f959c2647523cd6d5606", size = 8028, upload-time = "2026-10-20T22:44:33.334Z" },
+]
+
+[[package]]
+name = "fastmcp-slim"
+version = "3.4.11"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "platformdirs" },
+ { name = "pydantic", extra = ["email"] },
+ { name = "pydantic-settings" },
+ { name = "python-dotenv" },
+ { name = "rich" },
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9d/35/5f61dc70141df213eddff40bb2b8ccf68b02ecd666da6a1efab7c55b5694/fastmcp_slim-3.4.11.tar.gz", hash = "sha256:50e29b0830fb216aba437f35cfc7dd4bce7451f2f85f8786d6fff3da78d79386", size = 610380, upload-time = "2026-10-20T22:44:12.833Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/ff/71/614669d0a0d8dea87dc0dc684650bb36793027fc409197526532f587fcab/fastmcp_slim-3.4.11-py3-none-any.whl", hash = "sha256:24535dd843b3dcb3272b86c88fdf6ada5026c39ec5b56e2426557223c4d2c8bf", size = 785938, upload-time = "2026-10-20T22:44:10.921Z" },
+]
+
+[package.optional-dependencies]
+client = [
+ { name = "authlib" },
+ { name = "certifi" },
+ { name = "exceptiongroup" },
+ { name = "httpx" },
+ { name = "mcp" },
+ { name = "opentelemetry-api" },
+ { name = "py-key-value-aio", extra = ["filetree", "keyring", "memory"] },
+ { name = "starlette" },
+]
+server = [
+ { name = "authlib" },
+ { name = "certifi" },
+ { name = "cyclopts" },
+ { name = "exceptiongroup" },
+ { name = "griffelib" },
+ { name = "httpx" },
+ { name = "joserfc" },
+ { name = "jsonref" },
+ { name = "jsonschema-path" },
+ { name = "mcp" },
+ { name = "openapi-pydantic" },
+ { name = "opentelemetry-api" },
+ { name = "packaging" },
+ { name = "py-key-value-aio", extra = ["filetree", "keyring", "memory"] },
+ { name = "pyperclip" },
+ { name = "python-multipart" },
+ { name = "pyyaml" },
+ { name = "starlette" },
+ { name = "uncalled-for" },
+ { name = "uvicorn" },
+ { name = "watchfiles" },
+ { name = "websockets" },
+]
+
[[package]]
name = "flask"
version = "3.1.3"
@@ -1698,6 +1869,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
]
+[[package]]
+name = "httpx-sse"
+version = "0.4.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" },
+]
+
[[package]]
name = "humanize"
version = "4.16.0"
@@ -1933,6 +2113,27 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" },
]
+[[package]]
+name = "jaraco-classes"
+version = "3.4.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "more-itertools" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" },
+]
+
+[[package]]
+name = "jaraco-context"
+version = "6.1.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" },
+]
+
[[package]]
name = "jaraco-functools"
version = "4.6.0"
@@ -1957,6 +2158,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/be/c7/71eb0d51cdcbb63b857c931aa4e9218b078b5d9f8ecd383b234f28fe7b0f/jedi-0.20.1-py2.py3-none-any.whl", hash = "sha256:0fb16d86c4a4c73c37ba518c77419975e30fcc620658a8d14fbb5720cdd34142", size = 4885980, upload-time = "2026-10-12T17:42:21.585Z" },
]
+[[package]]
+name = "jeepney"
+version = "0.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" },
+]
+
[[package]]
name = "jinja2"
version = "3.1.6"
@@ -2071,6 +2281,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/72/07/3c953fceee478798f6bdf7b39e0da0793f437e8b5f9e9e1360fba32fb25c/jsonpointer-3.12-py3-none-any.whl", hash = "sha256:89f391335ee2719ae72149e1bf30a4d4c7c5d92f15a3f53faa8d20cc7c6c7f19", size = 10562, upload-time = "2026-11-11T09:34:21.711Z" },
]
+[[package]]
+name = "jsonref"
+version = "1.1.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/aa/0d/c1f3277e90ccdb50d33ed5ba1ec5b3f0a242ed8c1b1a85d3afeb68464dca/jsonref-1.1.0.tar.gz", hash = "sha256:32fe8e1d85af0fdefbebce950af85590b22b60f9e95443176adbde4e1ecea552", size = 8814, upload-time = "2023-01-16T16:10:04.455Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/0c/ec/e1db9922bceb168197a558a2b8c03a7963f1afe93517ddd3cf99f202f996/jsonref-1.1.0-py3-none-any.whl", hash = "sha256:590dc7773df6c21cbf948b5dac07a72a251db28b0238ceecce0a2abfa8ec30a9", size = 9425, upload-time = "2023-01-16T16:10:02.255Z" },
+]
+
[[package]]
name = "jsonschema"
version = "4.26.0"
@@ -2155,6 +2374,23 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e7/e7/80988e32bf6f73919a113473a604f5a8f09094de312b9d52b79c2df7612b/jupyter_core-5.9.1-py3-none-any.whl", hash = "sha256:ebf87fdc6073d142e114c72c9e29a9d7ca03fad818c5d300ce2adc1fb0743407", size = 29032, upload-time = "2025-10-16T19:19:16.783Z" },
]
+[[package]]
+name = "keyring"
+version = "25.7.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "jaraco-classes" },
+ { name = "jaraco-context" },
+ { name = "jaraco-functools" },
+ { name = "jeepney", marker = "sys_platform == 'linux'" },
+ { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" },
+ { name = "secretstorage", marker = "sys_platform == 'linux'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" },
+]
+
[[package]]
name = "lazy-object-proxy"
version = "1.13.0"
@@ -2564,6 +2800,31 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/41/09/5b161152e2d90f7b87f781c2e1267494aef9c32498df793f73ad0a0a494a/matplotlib_inline-0.2.2-py3-none-any.whl", hash = "sha256:3c821cf1c209f59fb2d2d64abbf5b23b67bcb2210d663f9918dd851c6da1fcf6", size = 9534, upload-time = "2026-05-08T17:33:32.055Z" },
]
+[[package]]
+name = "mcp"
+version = "1.30.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "anyio" },
+ { name = "httpx" },
+ { name = "httpx-sse" },
+ { name = "jsonschema" },
+ { name = "pydantic" },
+ { name = "pydantic-settings" },
+ { name = "pyjwt", extra = ["crypto"] },
+ { name = "python-multipart" },
+ { name = "pywin32", marker = "sys_platform == 'win32'" },
+ { name = "sse-starlette" },
+ { name = "starlette" },
+ { name = "typing-extensions" },
+ { name = "typing-inspection" },
+ { name = "uvicorn", marker = "sys_platform != 'emscripten'" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ba/93/0142dc84a666daf8ad51a34268f34c12fd6fda4f3810c4be2504eecc8212/mcp-1.30.0.tar.gz", hash = "sha256:445414625fce5c295faa505bb11bacece661ab6f4028d57c935db57820b7a3e4", size = 680511, upload-time = "2026-09-07T14:34:15.845Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/f5/f4/e58bc33317c92a0203664daaf00bf6f41166cc0149e5d6870a03f7cd004a/mcp-1.30.0-py3-none-any.whl", hash = "sha256:666edb5009503e1047c9d60346a756f94b261f05cc2625f23d41c728ffc484d0", size = 234581, upload-time = "2026-09-07T14:34:14.266Z" },
+]
+
[[package]]
name = "mdit-py-plugins"
version = "0.7.0"
@@ -2999,6 +3260,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/18/0d/1885a8307b845e5b44ddf568939900819653e0168356ab94ed670b0e7ed0/mutmut-3.8.1-py3-none-any.whl", hash = "sha256:e424d72c0557b6a52a721d9b71e2d18eb9590af746db0bbad50b71e8d4d8ed78", size = 82376, upload-time = "2026-10-28T07:54:36.982Z" },
]
+[[package]]
+name = "mutmut-mcp"
+version = "0.1.0"
+source = { git = "https://github.com/wdm0006/mutmut-mcp#7aeef920d3bcc05ce99a829ed8de9dcfb283faf3" }
+dependencies = [
+ { name = "fastmcp" },
+ { name = "mutmut" },
+]
+
[[package]]
name = "mypy-extensions"
version = "1.2.0"
@@ -3088,6 +3358,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/48/7f/c2d1b436b6e7cfebac140c2579a298344b85f2991a2ce5c3615cefb29400/numpy-2.5.4-cp315-cp315t-win_arm64.whl", hash = "sha256:7a14a461d9340f1b46b8648578aed9cdb8b3b018a8fac6c1dde2c9192a01a87f", size = 10883718, upload-time = "2026-10-10T20:05:28.547Z" },
]
+[[package]]
+name = "openapi-pydantic"
+version = "0.6.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pydantic" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/2b/32/0c9bd3e4e847cd6117b64dbef4cf810faa9cdbd6689323b811569cc8a1b8/openapi_pydantic-0.6.0.tar.gz", hash = "sha256:11f3ac6ad41521fc156381ec587246b0c0ecea523bd9565ed74d3e7fac9d91cd", size = 1667049, upload-time = "2026-10-02T12:20:00.508Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/31/57/9d3eb823a03c33e9d4c810e6a3276e3a6dc820a7fdc0961db2c04e613817/openapi_pydantic-0.6.0-py3-none-any.whl", hash = "sha256:8bd6d548a250da37db96f461bcd52b3468e8834ad7d6db14b00a6b2ebbd163a3", size = 96931, upload-time = "2026-10-02T12:19:58.917Z" },
+]
+
[[package]]
name = "openapi-schema-validator"
version = "0.9.0"
@@ -3134,6 +3416,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c0/da/977ded879c29cbd04de313843e76868e6e13408a94ed6b987245dc7c8506/openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2", size = 250910, upload-time = "2024-06-28T14:03:41.161Z" },
]
+[[package]]
+name = "opentelemetry-api"
+version = "1.45.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/19/14/cf3a6b6448849656e1e5f4e2101f3988dc6789724050096aba1bc5ccf592/opentelemetry_api-1.45.2.tar.gz", hash = "sha256:d7b0008e21e5fd61cbe7a7f34ca3b6905a78ab677d9bc395e299359c75cda6b4", size = 73277, upload-time = "2026-10-29T17:29:37.888Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/a9/2d/610db864d7513158cebb8f36cc88e67e73bd6c7aa1905ef18d89e1281c81/opentelemetry_api-1.45.2-py3-none-any.whl", hash = "sha256:2cbd737377e710ee740d261c94e93fa0f08e6d037c18f17513cc0a2d94479a34", size = 60335, upload-time = "2026-10-29T17:29:06.955Z" },
+]
+
[[package]]
name = "ordered-set"
version = "4.1.0"
@@ -3566,6 +3860,31 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6d/18/83376915176eb058cb86470eb7396388a13350b09a8f233a79303bbcbc5b/pure_eval-0.2.4-py3-none-any.whl", hash = "sha256:96cae060a313cfaad51bb761278bfb0e62dc0248d9315a81173752dc546cd37a", size = 11893, upload-time = "2026-09-10T21:41:21.532Z" },
]
+[[package]]
+name = "py-key-value-aio"
+version = "0.4.6"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "beartype" },
+ { name = "typing-extensions" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/ca/99/c346e3474853801ec5ecf4c3ee60cfc6060327ebc31424df4e346620dde4/py_key_value_aio-0.4.6.tar.gz", hash = "sha256:267c03c3e24cb99d3097612f8a5cfd8e11785c6a2975e272db0e303ea1850bfd", size = 112876, upload-time = "2026-09-18T15:29:52.002Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/9a/46/12871550fb0613d149144bb9d41f22680d5f46717bc45a76eb5798c20c3f/py_key_value_aio-0.4.6-py3-none-any.whl", hash = "sha256:820b30c7959fc738ea3d00160cdb10cd258f51e38737b2add530defeea1ded77", size = 171289, upload-time = "2026-09-18T15:29:50.665Z" },
+]
+
+[package.optional-dependencies]
+filetree = [
+ { name = "aiofile" },
+ { name = "anyio" },
+]
+keyring = [
+ { name = "keyring" },
+]
+memory = [
+ { name = "cachetools" },
+]
+
[[package]]
name = "py-partiql-parser"
version = "0.6.3"
@@ -3847,6 +4166,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/1b/ac/e5a5c90e26edc068408281c6c2c1c8fbb6aae6efd1b312cb2cdd629726c0/pyjsg-0.12.5-py3-none-any.whl", hash = "sha256:6cf60375f203dd275db24fb73b0874b6e1f7a90ef3eaa1e4dcded79842a56fcb", size = 81237, upload-time = "2026-10-11T04:52:28.978Z" },
]
+[[package]]
+name = "pyjwt"
+version = "2.15.1"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" },
+]
+
+[package.optional-dependencies]
+crypto = [
+ { name = "cryptography" },
+]
+
[[package]]
name = "pymdown-extensions"
version = "13.1"
@@ -3903,6 +4236,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/38/bb/d215ee7c73b61497b28a5503f9f53523f294fcc936762b7caf90e0c1c2b5/pyparsing-3.3.3-py3-none-any.whl", hash = "sha256:ece8c00a69cf01b45d0b1dedabb469c90d8caf996d4fda40f147627a122849a4", size = 126420, upload-time = "2026-09-20T20:59:04.025Z" },
]
+[[package]]
+name = "pyperclip"
+version = "1.11.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/e8/52/d87eba7cb129b81563019d1679026e7a112ef76855d6159d24754dbd2a51/pyperclip-1.11.0.tar.gz", hash = "sha256:244035963e4428530d9e3a6101a1ef97209c6825edab1567beac148ccc1db1b6", size = 12185, upload-time = "2025-09-26T14:40:37.245Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/df/80/fc9d01d5ed37ba4c42ca2b55b4339ae6e200b456be3a1aaddf4a9fa99b8c/pyperclip-1.11.0-py3-none-any.whl", hash = "sha256:299403e9ff44581cb9ba2ffeed69c7aa96a008622ad0c46cb575ca75b5b84273", size = 11063, upload-time = "2025-09-26T14:40:36.069Z" },
+]
+
[[package]]
name = "pyroaring"
version = "1.2.0"
@@ -4223,6 +4565,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" },
]
+[[package]]
+name = "pywin32-ctypes"
+version = "0.2.3"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" },
+]
+
[[package]]
name = "pyyaml"
version = "6.0.3"
@@ -4577,6 +4928,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0b/35/1cceccc5fcb50fa2ed53e2aa278cd032f3902682a73e763fb1ac3be8e6fa/rich_argparse-1.8.0-py3-none-any.whl", hash = "sha256:d2a3ce7854654e2253c578763ab0a32f05016f23a55fadba7b9a91b6c0e92142", size = 25616, upload-time = "2026-05-01T15:18:42.395Z" },
]
+[[package]]
+name = "rich-rst"
+version = "2.2.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "pygments" },
+ { name = "rich" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/cf/0e/faf7c7e36630561e3e9611730c47510cd972dd5fde8f95941ff78f76accd/rich_rst-2.2.0.tar.gz", hash = "sha256:b1e6a67f8f694a6f36035624bf73e2b1a0a4be13edaf3ba5e654d9758b61073a", size = 326270, upload-time = "2026-09-27T19:12:42.163Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/d1/4e/716b42bf066e180c1235783b46f095a5d871d05dca7006d2295aac29c491/rich_rst-2.2.0-py3-none-any.whl", hash = "sha256:1ea1c43813dc4a8d86475fce27ffb74e0ada3d4656c9f8130ba2a59868055fe9", size = 279274, upload-time = "2026-09-27T19:12:40.548Z" },
+]
+
[[package]]
name = "roman-numerals"
version = "4.1.0"
@@ -4718,6 +5082,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" },
]
+[[package]]
+name = "secretstorage"
+version = "3.5.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "cryptography" },
+ { name = "jeepney" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" },
+]
+
[[package]]
name = "semver"
version = "3.1.1"
@@ -5157,6 +5534,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/92/68/e351109f7732f193133c67fbf104178f79b534a925db8f69c074fd190c2c/sqlglot-30.31.0-py3-none-any.whl", hash = "sha256:51a876a43edecf8ee6fd4c872cf6713d899c4234256bba0be96a8c7c8d8781e8", size = 811981, upload-time = "2026-11-10T18:02:49.108Z" },
]
+[[package]]
+name = "sse-starlette"
+version = "3.7.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "anyio" },
+ { name = "starlette" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/41/43/a232e3cd804293bd4f6d01e8895bc0f6f83ea148d712baf9d783c7d93d23/sse_starlette-3.7.0.tar.gz", hash = "sha256:326416da2ab52e9a41b3f409d3ff897eda45aa68c9f3b7b17d0913aab8d219c3", size = 43869, upload-time = "2026-11-07T11:15:49.389Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/6a/11/25cabac0a24fa76f58637d8671b549c90d818d69d26beecf0c59a3b186e0/sse_starlette-3.7.0-py3-none-any.whl", hash = "sha256:f1d3136caa12d129284fca3d2ded79bc9af943320b6472712955fea52a6d3eea", size = 18432, upload-time = "2026-11-07T11:15:47.86Z" },
+]
+
[[package]]
name = "sssom-pydantic"
version = "0.6.10"
@@ -5187,6 +5577,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/7b/ce1eafaf1a76852e2ec9b22edecf1daa58175c090266e9f6c64afcd81d91/stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695", size = 24521, upload-time = "2023-09-30T13:58:03.53Z" },
]
+[[package]]
+name = "starlette"
+version = "1.10.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "anyio" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/48/bb/3d8757f21e9a53d9035c9eb039a777047139a16bfdbedfc7a58241c3d914/starlette-1.10.0.tar.gz", hash = "sha256:b8399098d4c1e3522734f7235dc79b1fc967f96cf139c81e0fa30479e0685f9b", size = 2734356, upload-time = "2026-10-24T15:04:12.342Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/88/2e/947cfe02def9371c07cd5bf5b6b61c829e9aecb9904db02a66cfafbc1eec/starlette-1.10.0-py3-none-any.whl", hash = "sha256:478c0ab3b4ced6e82719849c118fa406f0278a3499c1a898cd15cd31bdc38ce4", size = 80571, upload-time = "2026-10-24T15:04:10.371Z" },
+]
+
[[package]]
name = "strictyaml"
version = "1.7.3"
@@ -5483,6 +5885,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/0a/66/9a06c2227fedfd4919f0eaaa8b94377d9730882a7e4f95823c9a331ab5e1/tzdata-2026.6-py2.py3-none-any.whl", hash = "sha256:0a10eec6e13ef16b627fabed9a5f2ba61a62b1c67f327305f4e7e412a70b4aec", size = 348210, upload-time = "2026-10-28T09:57:07.276Z" },
]
+[[package]]
+name = "uncalled-for"
+version = "0.6.2"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/a3/60/993a9757dc4a41727351828dc784e66fd467ed81d46cbec25d44cae69657/uncalled_for-0.6.2.tar.gz", hash = "sha256:58835db667a85408c7b6d6ad8dfdd74d7713d3be669b46396ba3d2c6a889242d", size = 73820, upload-time = "2026-11-19T01:15:30.596Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/4e/fb/81dabcc9c8f4200eb5eaa600458978303f1a84ffa5ed356f70bf6c5d1f04/uncalled_for-0.6.2-py3-none-any.whl", hash = "sha256:bcac08897ceb7113b52e6c8070eead4da1610ae23e6e0d0e9f2a6c4bba7905cb", size = 20127, upload-time = "2026-11-19T01:15:29.148Z" },
+]
+
[[package]]
name = "unidecode"
version = "1.4.0"
@@ -5510,6 +5921,19 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" },
]
+[[package]]
+name = "uvicorn"
+version = "0.54.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "click" },
+ { name = "h11" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283, upload-time = "2026-09-25T06:52:37.601Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427, upload-time = "2026-09-25T06:52:35.829Z" },
+]
+
[[package]]
name = "validators"
version = "0.36.0"
@@ -5573,6 +5997,45 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/c0/aa/7a7210433e000f2491ddec7756d312009b0a47847e5c3457ccb03852cfdf/watchdog-7.1.0-py3-none-win_ia64.whl", hash = "sha256:0cfc5a6b3d4e76fc3e18fd371ee79d57945004da4105b589f71bb16966fd3e67", size = 103256, upload-time = "2026-11-05T09:33:20.159Z" },
]
+[[package]]
+name = "watchfiles"
+version = "1.3.0"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+ { name = "anyio" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/b3/68/e6aa0b77d217b31f8f486ec0cdfe5e00e6e38dc0be657e7d85819b9faf0a/watchfiles-1.3.0.tar.gz", hash = "sha256:99aee4a07847c06820765fd7b1b49ceac4f3f711ccb7d104655a33231de1c207", size = 108119, upload-time = "2026-09-21T09:08:55.664Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/68/fa/c0b840d5d8bafe640925408ce11095948c8945bb61be1e67d0ab629b872a/watchfiles-1.3.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:000b9688fc8133037a8b075c8ebf98f32844ff8964dda61e85c1db547dafc441", size = 399703, upload-time = "2026-09-21T09:08:13.518Z" },
+ { url = "https://files.pythonhosted.org/packages/c7/8a/894799b485fe9473ad10422a0d9668e53fb78e3a2b5cc6061159572844ad/watchfiles-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:bbc1198edfdc90fda0600f825aa94150f428dfcbf8138746f55998e0e660d64c", size = 397647, upload-time = "2026-09-21T09:08:15.16Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/61/f277cd5ead05f5d37b98b41fbdd0c943ee0c04d49738f62e902cb291cf43/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4af9c464d410e8c44b58ddf1cbec7c3a02c1cb2c6c80df9c74659ba2fbda0ab5", size = 454994, upload-time = "2026-09-21T09:08:16.587Z" },
+ { url = "https://files.pythonhosted.org/packages/46/66/ea4c01382975e53b1ac1b0a0ea1b353416b8127fb6e04f5c1c313119aab6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb94e40b9a0db19636b5e3fed0a9803452ecc7381df13b385272abb779ee83ca", size = 460105, upload-time = "2026-09-21T09:08:17.961Z" },
+ { url = "https://files.pythonhosted.org/packages/31/c3/1a501b597817096c9831c255f6270c124ce0e2faeb74124be24dcc93b86d/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7398930a2b76b9dc67a4e6b8bfc7baf30946892a86f417a3920c07bdd6febc30", size = 493761, upload-time = "2026-09-21T09:08:19.327Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/16/77352d05e152c7786a016e9f1decdb4507e8aa5bcf3343c72142a0413d14/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:025b108f2d5cc2799cb941f79380df3e8b22bfb589c218e801767fe78d598c95", size = 577500, upload-time = "2026-09-21T09:08:20.723Z" },
+ { url = "https://files.pythonhosted.org/packages/95/fe/81001c55b24466fe9911135e50269fd3a29f43e446627a92adfab54553b6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9ef3cd403d756a304a0a08a112163c63da7ec72c8c9d083c98818291ff8f29", size = 469209, upload-time = "2026-09-21T09:08:22.346Z" },
+ { url = "https://files.pythonhosted.org/packages/8a/cf/e3ad894ed5909a7c44b0707e89bc25d19a3d78b3a7aec846119443b6ae56/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5768b49e426fd5b550b012c866db347cdf15c398ef98dc557b6e6b72fa74cd1", size = 458353, upload-time = "2026-09-21T09:08:23.7Z" },
+ { url = "https://files.pythonhosted.org/packages/7f/f5/2f106829e6e2bc91ad74614730f6df484202b9a4d20d927dbf2f08d37d44/watchfiles-1.3.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:d36c72fcdc08f143d87316a4c4c3b064e340da2c7383d39803950740ed418935", size = 468707, upload-time = "2026-09-21T09:08:25.167Z" },
+ { url = "https://files.pythonhosted.org/packages/15/37/2741ab776b978669c1f3bba1e573f1a184f727ab8b4d3a6d9c61090800ed/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c9865240e065a2247f4b5248528ae3d01e0e8ae071462c250c884dc16e3ee3", size = 632510, upload-time = "2026-09-21T09:08:26.532Z" },
+ { url = "https://files.pythonhosted.org/packages/48/64/db4a4684275fe9b696ee5118c194705c1423b3d42181c3f8ee70194d5c73/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:380f513d26cc2e598266b88d66456e07d67ef4be8f0f435a1154d9c44b43d509", size = 661723, upload-time = "2026-09-21T09:08:28.004Z" },
+ { url = "https://files.pythonhosted.org/packages/52/80/95ddb24f6a2f595c031c24432ab7b8aa7d6d1ada0c6d63a552166e06e8d1/watchfiles-1.3.0-cp310-abi3-win32.whl", hash = "sha256:509d9f74d2bec5c1f4cd868ddcfe0c9601f7ea53066dfcb19d8ede742c4de661", size = 277761, upload-time = "2026-09-21T09:08:29.335Z" },
+ { url = "https://files.pythonhosted.org/packages/48/a1/1c4a1b3c8030cd03206232612be8a6e8799093490c220b0fcb935d02b75a/watchfiles-1.3.0-cp310-abi3-win_amd64.whl", hash = "sha256:1acabde19b67e673274e89a04d613b7fb1f122e2c4b8ec9a581e59c149264b61", size = 290643, upload-time = "2026-09-21T09:08:30.45Z" },
+ { url = "https://files.pythonhosted.org/packages/82/e0/d2db577c632d26d08078c4390dd870b8555c4e558df25d22270f0166c71f/watchfiles-1.3.0-cp310-abi3-win_arm64.whl", hash = "sha256:d978cc1dd7ba44f5590d7de74f7a9c8be7262d9e469c2c0efb8bbfc2574321dc", size = 285510, upload-time = "2026-09-21T09:08:31.561Z" },
+ { url = "https://files.pythonhosted.org/packages/4f/55/a122743607939429ab6a7272672f49303aa02bbd34c3f06676765a3861de/watchfiles-1.3.0-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:e0204f90677b7fb8d3a824279a71a01166bea9f6e53923b8056592d87b33b0b6", size = 395978, upload-time = "2026-09-21T09:08:32.747Z" },
+ { url = "https://files.pythonhosted.org/packages/3e/9d/0ee1b0ea42acfe72afa2484f0222e2386dd09992e9034a81e63cc3cdab0a/watchfiles-1.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:7c1125e99f84934a92996ae343501285617373acd66f16b59fd5654b27ad8d80", size = 387997, upload-time = "2026-09-21T09:08:33.953Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/f2/4319c6b683e8bd28465dfc53c750617a2a07f072eb2e3ab0182ccf812229/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a85e243c87109d9f48a2e0d10d6abdee8768cc114848b32281ec83d7f81d86ef", size = 448067, upload-time = "2026-09-21T09:08:35.104Z" },
+ { url = "https://files.pythonhosted.org/packages/b7/71/9f37ac192a51a24de3a2d7fe9aac1e99b2b2668300674ff7a9f85d95beb1/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:dd538c71766c59c732e2c5cedc39323c99c11897b99f93eac773438e54af206a", size = 452881, upload-time = "2026-09-21T09:08:36.238Z" },
+ { url = "https://files.pythonhosted.org/packages/34/07/1224667c32a4a96876ddd642da69733cbfe6a15897db0865aec5a882cebc/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da213822ec9082b62cadf3008e07a8c690ec7a3c89f6dc5e55a8cf07b56a1b26", size = 484802, upload-time = "2026-09-21T09:08:37.717Z" },
+ { url = "https://files.pythonhosted.org/packages/bd/4d/7f29cd87e1f826df86fc0418d0b8f370ca0311c29c618406dc492aad9951/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4593392e87669836670f87d62fc754059eaeb0e6157af44060a46558cebe5a3", size = 567522, upload-time = "2026-09-21T09:08:38.94Z" },
+ { url = "https://files.pythonhosted.org/packages/e3/ac/3aa1f84abd4dbbfbae99a817dd931f77b439261c911189cad4c30fdfd470/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:26b81bc515a0f03f66a69a6fea86eea54c8eca4173c46dd7222569f79dd2f977", size = 461997, upload-time = "2026-09-21T09:08:40.142Z" },
+ { url = "https://files.pythonhosted.org/packages/82/1c/27c66764d5b0dbb70a36770a32eca217eba739c28b24e8583a035087a3d7/watchfiles-1.3.0-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ca39661934749df580d89f3dde266f2af7c6968f4c21c72ce2dc387d299e6aa7", size = 451856, upload-time = "2026-09-21T09:08:41.305Z" },
+ { url = "https://files.pythonhosted.org/packages/1e/b3/a45827084d953c1d0dd3d75abf85dcac26a0f5d115ad2fe0b7e9e931573b/watchfiles-1.3.0-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:04c5500faa725d69a99b0f63750e80795e455bb951849e9c98a811b4324997c4", size = 459889, upload-time = "2026-09-21T09:08:42.58Z" },
+ { url = "https://files.pythonhosted.org/packages/7a/3d/07740d23ce2bb23419bb24055c435a9abe5fe03efdaea1ba0720b831aab6/watchfiles-1.3.0-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:ec3cd4bb181a7b6329134204c2277c7144ab5f62cce9c5335d6f901ee7ce8d6a", size = 624482, upload-time = "2026-09-21T09:08:44.064Z" },
+ { url = "https://files.pythonhosted.org/packages/91/46/9540f8123eaec86df989f26d7704cad0448d9fb7271763a4f8f9fe23617d/watchfiles-1.3.0-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:e5871d4a7f788a9f64fb06f5793389d29d171e5c32d37c3ebfd520b017cc7694", size = 653724, upload-time = "2026-09-21T09:08:45.257Z" },
+ { url = "https://files.pythonhosted.org/packages/25/9d/b9deb52f1e4528ffe44338c5ebab95c32ed79203082da661de50288e130a/watchfiles-1.3.0-cp314-cp314t-win32.whl", hash = "sha256:81c989d7267bfb7b676cd32c83bec97914e396870983cbf493e29415037bff5e", size = 270863, upload-time = "2026-09-21T09:08:46.518Z" },
+ { url = "https://files.pythonhosted.org/packages/59/59/8b1881afccbafda06248be4185e28034dd73c80c85e4da24f2cb0c1321eb/watchfiles-1.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:d754e049006e81a9be49dc78ba7802081cee426e968cbc63ee19e164fa2d0e39", size = 286879, upload-time = "2026-09-21T09:08:47.874Z" },
+ { url = "https://files.pythonhosted.org/packages/46/dd/41c12ae9745d5d72f709333287679d71cbba68ad7678fed872207bba94f4/watchfiles-1.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:a5b631db08fd3032db57ec6a92d1777cd60f88803287e4e941a4858c53aab1e0", size = 276924, upload-time = "2026-09-21T09:08:48.976Z" },
+]
+
[[package]]
name = "wcwidth"
version = "0.9.4"
@@ -5618,6 +6081,115 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/e2/cc/e097523dd85c9cf5d354f78310927f1656c422bd7b2613b2db3e3f9a0f2c/webcolors-25.10.0-py3-none-any.whl", hash = "sha256:032c727334856fc0b968f63daa252a1ac93d33db2f5267756623c210e57a4f1d", size = 14905, upload-time = "2025-10-31T07:51:01.778Z" },
]
+[[package]]
+name = "websockets"
+version = "17.4"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/75/79/590c5f0cc7d1661195aeefccb0e342b79f4c77d368d526d028821a390332/websockets-17.4.tar.gz", hash = "sha256:09d3939b70df0c8ad2da145ebd50978bccaee1d565e838b0e7921a34d38571eb", size = 191391, upload-time = "2026-10-30T15:10:35.873Z" }
+wheels = [
+ { url = "https://files.pythonhosted.org/packages/cd/58/eaf27fce59e3f5b9997cd7c77749ef214e077daeffac95479be6ff9f5cfa/websockets-17.4-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:f24bed98595b9d854da8d6debaf82754abf3e8fe63ec90505ef560566814df7e", size = 218769, upload-time = "2026-10-30T15:07:42.558Z" },
+ { url = "https://files.pythonhosted.org/packages/13/8e/287eca6d399a953253492189153976676994dfba1b7e82f0f992d14c1404/websockets-17.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:156221ad7638b5abccefd6f798916bf3f8ffd6ac51480eb4ba5b05df914ccbf3", size = 216454, upload-time = "2026-10-30T15:07:43.874Z" },
+ { url = "https://files.pythonhosted.org/packages/b3/65/098fd4d221d691395492fcba88c895dde19a51d371b06b8d0be71d3f781b/websockets-17.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:74e46409666c358b93fce0d8c6f4cb107d8dd7cba1302a38d49adf6fa8193c7e", size = 216715, upload-time = "2026-10-30T15:07:45.409Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/29/9f9833bc83c1bf2e5597d1ef374d3ccd6467b9b45400cff399c85331ef6c/websockets-17.4-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:e3e80cad85ba68c07c1e8d4ccc9cf51e63bf6a73d6bf631640be947bd72b6b63", size = 226096, upload-time = "2026-10-30T15:07:46.929Z" },
+ { url = "https://files.pythonhosted.org/packages/ef/75/6a95f93661bba2f1a4ec82eb26c6d30e6ba253569ecabb7a675139db6051/websockets-17.4-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:25eb661d8ba7e2b85ebb4e7a1bb9f6a3280910ed10b84330a7abfb5ed32d2764", size = 226382, upload-time = "2026-10-30T15:07:48.332Z" },
+ { url = "https://files.pythonhosted.org/packages/80/b0/b0d9427dde0609e2727a9a966fb5fc5c6ef6aa7eba4cfc98791f8b7f1b75/websockets-17.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c74954409ea5789f7a60daa5f94580c00ad0b60b20751c47b35c4784877b5057", size = 227617, upload-time = "2026-10-30T15:07:49.799Z" },
+ { url = "https://files.pythonhosted.org/packages/3b/00/fb1016d850d61676d1732434e2361d116c1de369078e59d21f4713115a19/websockets-17.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:47beedf07a706140b50786457e082497ee6b5970c41e49bcd1572ee3f4c73f80", size = 229054, upload-time = "2026-10-30T15:07:51.258Z" },
+ { url = "https://files.pythonhosted.org/packages/77/c4/bb666e196541e42237fdb625303ad099005cc835c4e2e0a5d97bb9c4185c/websockets-17.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e282f35ece8895b9fa5af406fc6a4cb42f01e1009525cf46641de584781c83b", size = 228243, upload-time = "2026-10-30T15:07:52.975Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/f1/32ae0f45d8464e1606ef3f6a54318d8754b27889de18bb1b2176a19bea39/websockets-17.4-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:361f9a21a33191d5287e7a75f508fc3b9bf47aaffb0241198d6b7be3384f294e", size = 226995, upload-time = "2026-10-30T15:07:54.574Z" },
+ { url = "https://files.pythonhosted.org/packages/f8/9f/5680151ef10e0bfb6f3bf09a992ce6c3e93bbb84fc8ec5d03ad570ac6532/websockets-17.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4073ab7cefe5a2e7dc368c1f01eb27716dc8ea8f1c69e9b7725ebce05aa53cf7", size = 224244, upload-time = "2026-10-30T15:07:56.158Z" },
+ { url = "https://files.pythonhosted.org/packages/e2/66/409117b6882b91960598d62b95fac33d70a7aa2e1e1e854c6b64e76712f6/websockets-17.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a50bd9b44dd9b7b845802030b4d88610c01c7f95571b4cbd3a1aeec17a34c9fd", size = 227058, upload-time = "2026-10-30T15:07:57.539Z" },
+ { url = "https://files.pythonhosted.org/packages/35/e6/34cab5913bea9d011ae2592a7672340179ff14fb98e889e5e41b071d8134/websockets-17.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:ad2e9e152bec6c4f7e4cdb4adcea6b708400090eec04bfe5ee5b30d368ffd721", size = 225654, upload-time = "2026-10-30T15:07:59.044Z" },
+ { url = "https://files.pythonhosted.org/packages/a6/b0/6cfe18e12f23618a3d69723c828037c1282fee7ffce006ae7b5cc0e58ca4/websockets-17.4-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:5830f94a7190a33e55d06d89242ea951ee2009c3a27fc31538f3ac9386b27edd", size = 226425, upload-time = "2026-10-30T15:08:00.492Z" },
+ { url = "https://files.pythonhosted.org/packages/c9/71/14d31ea34bafc07c6d96b362a44b301b9a0f2fb260490a8a6cb7db682bc4/websockets-17.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:c0cc2cb47face4dae64bd70a9268dbebddb3e48c1b48cf6d7b6ac1471609fd88", size = 227532, upload-time = "2026-10-30T15:08:01.932Z" },
+ { url = "https://files.pythonhosted.org/packages/28/69/df3398ca1320a8b76e57ee41053d5c09955383cf60888ffa5771f51a4cc8/websockets-17.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:d0b6a9bd86f865ebc9d847a1eb0a655a0e2b447cbcde743dda97c4298415cc0b", size = 225090, upload-time = "2026-10-30T15:08:03.328Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/94/06afe778cf9893e9ac10209c0d4c70aaf0b72289245a8b7efb92cc80927c/websockets-17.4-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:7a7e9321822ca59ba56fa8744c3e3f01ef74d547aafe0d383d000d7a4d11eb79", size = 226041, upload-time = "2026-10-30T15:08:04.714Z" },
+ { url = "https://files.pythonhosted.org/packages/c6/7a/28e9822dbf0bc63341dc45d8d5f1f1f7f375991fc8895bc731105fbde371/websockets-17.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:de29795f23833cdfb68ba45c8a5cc0ec49778ea5deb670188f4465bba5930d25", size = 226323, upload-time = "2026-10-30T15:08:06.103Z" },
+ { url = "https://files.pythonhosted.org/packages/5a/d8/606c9c404e82731e4520f6397c8c16549e9cbd39ac583e7ea7d8972a6ae7/websockets-17.4-cp313-cp313-win32.whl", hash = "sha256:37ed7782972545e67d97ef9a172e9499d25459b1e32cb99d1c2a30c572f2441b", size = 219248, upload-time = "2026-10-30T15:08:07.617Z" },
+ { url = "https://files.pythonhosted.org/packages/60/a1/17e507dadbfccb755758ba977b165fec92924dfc4a8d2573b4fc1561b0cd/websockets-17.4-cp313-cp313-win_amd64.whl", hash = "sha256:203e42eb77e06f85778a908b34c5ad032f34e98a68ef33920d6685c1e65a9420", size = 219560, upload-time = "2026-10-30T15:08:09.108Z" },
+ { url = "https://files.pythonhosted.org/packages/01/ba/4f51fe8db73889ea3ecfdeee283d4ac3b7ad7a609c0e4c1fc6de04078021/websockets-17.4-cp313-cp313-win_arm64.whl", hash = "sha256:d8255971f6789cf0cb365e3e22b9b166eba978548a26241eaa5b7af0440d885a", size = 219496, upload-time = "2026-10-30T15:08:10.574Z" },
+ { url = "https://files.pythonhosted.org/packages/1f/5b/085530961668661300de7248f9bf3ba84e98f47875f0fcc87637d024f82b/websockets-17.4-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:d0f943b53bb1d7389d72a5380078670e5fcc34a1328142148b72d20fd82031c7", size = 218810, upload-time = "2026-10-30T15:08:12.14Z" },
+ { url = "https://files.pythonhosted.org/packages/f2/fb/5a29e9569f74988822b7bc076287dd7f9f5b68a2070cf5eb4cfada5863d8/websockets-17.4-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:2fd562372d4afaf18d1c7f1c68e39bd32818bb16d5f57f778a4a4ed88eedcaae", size = 216506, upload-time = "2026-10-30T15:08:13.561Z" },
+ { url = "https://files.pythonhosted.org/packages/df/73/eb8544367c13cd87df39d41b74853741fb7182e2a4bb659ad7d36f9066c8/websockets-17.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:a0457561e8a87a6a3b509e21c35084d7d207ced262139de3f738e9676a476bbd", size = 216719, upload-time = "2026-10-30T15:08:15.08Z" },
+ { url = "https://files.pythonhosted.org/packages/24/fe/c3707e426d9bd779dfad9f319b2b431e5e60d563d7547d3780cbd1646028/websockets-17.4-cp314-cp314-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:7b4ee34acefa9ba1333f02b817982f1e4127f6f9b975017a707c4e738d7f9fb9", size = 226098, upload-time = "2026-10-30T15:08:16.581Z" },
+ { url = "https://files.pythonhosted.org/packages/c1/24/d203439f909ffbf37260ca28a3432381412d2e79912ebf98be572e63055e/websockets-17.4-cp314-cp314-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:ef03fb4b3685df0ed9dc6a074a090f030345cad2c23c61818da3e08e693a9caa", size = 226449, upload-time = "2026-10-30T15:08:18.29Z" },
+ { url = "https://files.pythonhosted.org/packages/4b/17/3ae5d9486a170a222dd0437b4f2bc02ecf324177498a942dcac200e65d05/websockets-17.4-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5e970d3996c597186c9dbb6e77a8048737a834ac03cb13186a54999e0e1c31e", size = 227695, upload-time = "2026-10-30T15:08:19.692Z" },
+ { url = "https://files.pythonhosted.org/packages/34/37/1307a8e30a961726f2350982fc01b0b3498db29b030c78c566e65ba64487/websockets-17.4-cp314-cp314-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6afc32a7a1a40b071070419537df3559d26907086556769683fa97b7b6e90fd5", size = 229069, upload-time = "2026-10-30T15:08:21.682Z" },
+ { url = "https://files.pythonhosted.org/packages/9c/bd/4c6167744e202d20a3c770bcde8fae5d427317b1885712c2e02718cd8133/websockets-17.4-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5da7f7f1ecf866186f195f7ecc374c915e1838496b86f2f3b41da27c798a25c3", size = 228296, upload-time = "2026-10-30T15:08:23.371Z" },
+ { url = "https://files.pythonhosted.org/packages/14/b3/3438eee41b39afb44c3c6e45dd686b88a7f8d742b96877b77de4fe0e0255/websockets-17.4-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:043945a43873c7c11a3f75238d475c95f902e967de329eb8763704546cfaa6b6", size = 227045, upload-time = "2026-10-30T15:08:24.749Z" },
+ { url = "https://files.pythonhosted.org/packages/46/39/0bac9052d402efaad1b202ff2f1a57b6334aa624d20f509312765601b507/websockets-17.4-cp314-cp314-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:927d602fb8fb595a69e6b1ae998a6ec19851d0fb9391759456e6425090996647", size = 224297, upload-time = "2026-10-30T15:08:26.275Z" },
+ { url = "https://files.pythonhosted.org/packages/a1/7f/5809f6ba5a7d1065301800daf605bdb661a9db5ef868ed189dbf691122ac/websockets-17.4-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:769e4f7b090e10cae4c082e3e39b9458ec2444505df84953c16c0468b16539c4", size = 227169, upload-time = "2026-10-30T15:08:27.703Z" },
+ { url = "https://files.pythonhosted.org/packages/fc/f7/c8f6771e7cc2a038cf78bb12d04d29dbbbabf2a84d76999bcc94532f3020/websockets-17.4-cp314-cp314-musllinux_1_2_armv7l.whl", hash = "sha256:03ddf4eb76ced8725dcbb000ff60b0fb8d7330ed011f6dc725a1925c5fe02051", size = 225632, upload-time = "2026-10-30T15:08:29.441Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/2f/9104786d952978ca2d011321109e35c05be24c3937ef2a62b4d6e25f7515/websockets-17.4-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:94622cf24e80d03054ceb69a06a86c950e12fe993ed06b933f7895232cacbc3e", size = 226415, upload-time = "2026-10-30T15:08:30.962Z" },
+ { url = "https://files.pythonhosted.org/packages/ae/a7/2e1606b3d616b86bb6c70a62d5f87fc9a3dbd81f0bbe24c4706ace76751d/websockets-17.4-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:bc581579885728bb54ef15000295c20e6a64a200b1b73d5cfca71879a3eb3e1f", size = 227594, upload-time = "2026-10-30T15:08:32.443Z" },
+ { url = "https://files.pythonhosted.org/packages/67/2d/78ca44e2be6d397aee63a4e68d3a99a33bb14bb3405e94688bc5a2160db5/websockets-17.4-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:f42671900e0b602d05a074c52fd3d757c2f8e5c4f73f771794dae1aa4aea5f0e", size = 225146, upload-time = "2026-10-30T15:08:33.888Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/b6/6dad684e54fc8e09d70c9a095506db5c836207d8e846e3cb1e79424e3492/websockets-17.4-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:2b23dacf7a8a88afe92f77bcdbf4cf043a84692bac2f2da02f4011c687e19b48", size = 226095, upload-time = "2026-10-30T15:08:35.285Z" },
+ { url = "https://files.pythonhosted.org/packages/d9/e5/7b04bad7c11c1f92238429352defbac1406743338049afe1328c7a7f95c0/websockets-17.4-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:d48795c7d64ca1c42e3d61fcc6478cb15a19b6ad3f9c882bc8d8dee49d4a420f", size = 226395, upload-time = "2026-10-30T15:08:36.8Z" },
+ { url = "https://files.pythonhosted.org/packages/b1/16/4eb29c4b12f77b3139470063308ae1e6a0860b9073b1d27d56e8ce19c512/websockets-17.4-cp314-cp314-win32.whl", hash = "sha256:8588edf317fbe4f16a949bc5a6ccdd22b6e6fd5b4745d128bc40bccf31024ce5", size = 218784, upload-time = "2026-10-30T15:08:38.263Z" },
+ { url = "https://files.pythonhosted.org/packages/6d/85/2496a24de80c215b55a15f037bc368a2ed0ce7ecb1e909039ff4e76395de/websockets-17.4-cp314-cp314-win_amd64.whl", hash = "sha256:70293c25d79d1f084fdd665894481547575e429d2a75e8d72caec2f6b6b07af8", size = 219137, upload-time = "2026-10-30T15:08:40.112Z" },
+ { url = "https://files.pythonhosted.org/packages/9a/a3/cfe95d4eb569035e5a1d2e5cbeb4ccf0eb15a19e1ef2ad819a85960bf375/websockets-17.4-cp314-cp314-win_arm64.whl", hash = "sha256:adc30e5d064c9e33049e825408fa4e952f4e8a0cf4c329dc2ded72e1bb5bae08", size = 219017, upload-time = "2026-10-30T15:08:41.778Z" },
+ { url = "https://files.pythonhosted.org/packages/2b/dd/4143048a2b27303e7de8ef53d7d0ee3987734fbf93e09491d5a1b0a55187/websockets-17.4-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:bbb6bed49f083e2ec8ae0bb21c00c7c5171ab8e08a8b008c61a43fc6c944fdac", size = 219141, upload-time = "2026-10-30T15:08:43.184Z" },
+ { url = "https://files.pythonhosted.org/packages/a8/3e/a63b5f3a5c24be2898eb22a3093a56d4ac4cd24ae79e7ab7a1b971ed4611/websockets-17.4-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:c7a56140fcbf4662cb263754a754c651a16fe2196cae1fd232d18f89acd30c83", size = 216675, upload-time = "2026-10-30T15:08:44.682Z" },
+ { url = "https://files.pythonhosted.org/packages/d6/80/f238a10f0b23853c665c602a42784023995ac26674e480252c21af39f4f8/websockets-17.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:18d3710becc612ec6f0229425bd6495133dfcbf30f7b6569585acc8ed2f1b452", size = 216880, upload-time = "2026-10-30T15:08:46.304Z" },
+ { url = "https://files.pythonhosted.org/packages/d1/b1/899b69aba1e8d65ac5bfe90b97ada22a815f1dcd3840bf188104738aac33/websockets-17.4-cp314-cp314t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:00ad98e4bb0df7a6711c645eec16d5a9fdf99f0692bd4f5b6cb85a68677ba8df", size = 226462, upload-time = "2026-10-30T15:08:47.723Z" },
+ { url = "https://files.pythonhosted.org/packages/28/a8/598c2b2a22dfe14d352400df804db8b78a68c1498b3f775a658f07f21618/websockets-17.4-cp314-cp314t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:7849a096d6fdba3f2bdf849eb89f2d233a24400c92faa3efb25fa8dc615b706c", size = 226741, upload-time = "2026-10-30T15:08:49.314Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/ad/84a6de3cc3dd4ee336428c31f89a82e002299f83e10d967f0e9a5cbaf591/websockets-17.4-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e30550965c78223439815dbdf40c93eb8931dfbf12427110dcfcaeebef76feb6", size = 227914, upload-time = "2026-10-30T15:08:50.746Z" },
+ { url = "https://files.pythonhosted.org/packages/69/ab/1e191b97d60a934c071b7b73c39c5db39721a925cc53acd929c615f85550/websockets-17.4-cp314-cp314t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:725be1aff55f1c873a79cea0f612df9dff21a9d2c2e05783665b824b8aa7c24e", size = 230059, upload-time = "2026-10-30T15:08:52.596Z" },
+ { url = "https://files.pythonhosted.org/packages/f2/57/ef8a64732ef4367091ab05f9b22c1ee4c28657d4a0d088a644198eb5ded6/websockets-17.4-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b6c7f91520c29420639925fd1cecceac9ca07366b50c09ea24eac1cd116b4866", size = 228486, upload-time = "2026-10-30T15:08:53.967Z" },
+ { url = "https://files.pythonhosted.org/packages/48/1f/dbad3afcacf2c52d1fdf83b611d849598f0a047d7333ff476266532877b8/websockets-17.4-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6531540aa63e6864cb0615a3a5da70ae03191334507852a5385320bb7916df3f", size = 227216, upload-time = "2026-10-30T15:08:55.687Z" },
+ { url = "https://files.pythonhosted.org/packages/5b/47/59e325d6ca3e04ba5cc7a139d891ef83cad3fb460a183c8f359f378b69a9/websockets-17.4-cp314-cp314t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0a57b99f2e28922b5a2434317580fd23fdf118caff4c5b752f5452836426205a", size = 224755, upload-time = "2026-10-30T15:08:57.203Z" },
+ { url = "https://files.pythonhosted.org/packages/b8/b6/28a471318dfb0060b58c47d0a252cb5078528a1aa1fdf5e6e88e2a61cf18/websockets-17.4-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:8fd30d217a4275107667fe1d4f601e2d50c797d0727148cfbb384536d65f04d3", size = 227508, upload-time = "2026-10-30T15:08:58.597Z" },
+ { url = "https://files.pythonhosted.org/packages/ee/29/7485ac6fc4b342b43964811381f56276fec967cae6a851ae4b815613b10c/websockets-17.4-cp314-cp314t-musllinux_1_2_armv7l.whl", hash = "sha256:cc0cd4065a65ac98f3e2742ee9f314da9d224db49b07dc17d3a20cef0d79658d", size = 225961, upload-time = "2026-10-30T15:09:00.414Z" },
+ { url = "https://files.pythonhosted.org/packages/95/a6/520e1f7aa9644ddf58f5a51fc37f46e15293a68cd35b70d5b81b7331ca1e/websockets-17.4-cp314-cp314t-musllinux_1_2_i686.whl", hash = "sha256:71f76bb8e59be847330b8a8d3753f85397677e128f9f9692c215f8cf721cefc7", size = 226851, upload-time = "2026-10-30T15:09:01.979Z" },
+ { url = "https://files.pythonhosted.org/packages/e5/0d/444799abe7418f361da8c201c6858b64b4e8d47265f90851e5cb68b4d90f/websockets-17.4-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:c2db7b9e2d726f7e051bedb67abcf460b84b11437b2f9fdacda7e6931dace85b", size = 227869, upload-time = "2026-10-30T15:09:03.405Z" },
+ { url = "https://files.pythonhosted.org/packages/cb/86/6da20322a07364a0ab0787a2b0d4367770f59404fdab6e35122413a7f63e/websockets-17.4-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:332b88022d0acd47c44158bcd43f47f14eaddcd979247f242b1a33291c45cdd1", size = 225562, upload-time = "2026-10-30T15:09:04.891Z" },
+ { url = "https://files.pythonhosted.org/packages/2a/da/53e44d96238dc207011ad5d93b0540c58e1c680a86f48dc3fa8e5138b343/websockets-17.4-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:5cec3e7fbc911ff4fcf43b24aaf1b96906381ad36030f42897def3ff4b78729e", size = 226334, upload-time = "2026-10-30T15:09:06.42Z" },
+ { url = "https://files.pythonhosted.org/packages/18/08/82cea972a87f229ee1903e9adf74c17d1e6c4371d581ebb11ccbed4e3a7b/websockets-17.4-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:cbdf4b42f4514422051889424fd03bc269be8c5d751bf39ec29ad761590bd10d", size = 226767, upload-time = "2026-10-30T15:09:07.837Z" },
+ { url = "https://files.pythonhosted.org/packages/96/f3/ee6afff802e39e8d6ef29ee4bdc30f8272aaa38295c19a911fad91fe95fc/websockets-17.4-cp314-cp314t-win32.whl", hash = "sha256:16550e24ac22c27e0a728100d8de196b5f054bab6748ef9f0b168fb8b3c32e35", size = 218918, upload-time = "2026-10-30T15:09:09.31Z" },
+ { url = "https://files.pythonhosted.org/packages/11/56/b0b07917f9c85a4d24a37300f4ae888bfa7c20f08902dd6a5c61a0d1b26f/websockets-17.4-cp314-cp314t-win_amd64.whl", hash = "sha256:5ec906bb21b35e66cfdcc2ea0133f249f52b7d89e3e45ec8d2bd2d84a0255d7d", size = 219260, upload-time = "2026-10-30T15:09:10.832Z" },
+ { url = "https://files.pythonhosted.org/packages/19/a6/a63113909ea602d3d22b0f32a7a9950d8d18b725cfdce735eab9b5f5d198/websockets-17.4-cp314-cp314t-win_arm64.whl", hash = "sha256:c00f0c45851a2c61efffd03dde8fb4d594026b5cc18196c94e15460f1225f2ed", size = 219176, upload-time = "2026-10-30T15:09:12.444Z" },
+ { url = "https://files.pythonhosted.org/packages/ee/be/4032fde05e62d1c52edd250fda475e910cac0cd180e377690bff24147b5e/websockets-17.4-cp315-cp315-macosx_10_15_universal2.whl", hash = "sha256:8c51ac7b31578a7275650f580aac985940985b6d2e57247ac6d5e07f2c936cd6", size = 218740, upload-time = "2026-10-30T15:09:13.833Z" },
+ { url = "https://files.pythonhosted.org/packages/74/1c/b47f5786eb45599072411571651606fb22f6e825d522d627daad974ea636/websockets-17.4-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:023dc13bdc42a55e152acc81c39c90ec6cc4159ed17d23c6d73460fa2a73ab16", size = 216470, upload-time = "2026-10-30T15:09:15.394Z" },
+ { url = "https://files.pythonhosted.org/packages/70/98/0021f55de633617b4a8390cca17e700c9ba5d99323103128412dc2c15aca/websockets-17.4-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:6fd8cac4907fec8302726d49e6969acfffaf720bc39b1cee23f473b81f4dce01", size = 216695, upload-time = "2026-10-30T15:09:17.185Z" },
+ { url = "https://files.pythonhosted.org/packages/28/56/2c67389b47c4b9e9e9318d9b3e4bb52a2fef3749ff00b0b151a342c4d42d/websockets-17.4-cp315-cp315-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:4441d655a53185ef1b01b1d42ad9b3a96b905110a9219043e3fdaec855ba3b6e", size = 226134, upload-time = "2026-10-30T15:09:19.156Z" },
+ { url = "https://files.pythonhosted.org/packages/75/bd/ee3a679a10741a0cdd7aa8bdd6d03999069672826aa7a6495fdf589ed961/websockets-17.4-cp315-cp315-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3acc45494c08a4d27375a2c28474537f68910219f42624a08011221b04623d04", size = 226569, upload-time = "2026-10-30T15:09:20.587Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/42/b293a5d07099549797138d09fda5bf878d302b4980e61678981c8dfdc98f/websockets-17.4-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3b78b39377630a70aa2198e5f03be416e690364a046b1d8af80a9263733083d0", size = 227726, upload-time = "2026-10-30T15:09:22.015Z" },
+ { url = "https://files.pythonhosted.org/packages/2d/81/d048ff83c4d0fc386f6008529d12690d0dcb410f73570430f13fe4d543f0/websockets-17.4-cp315-cp315-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:6cf4c002268da5c11cae9083aec71ac943bce9d95e0f83c8bf9a95a3ce421fbb", size = 229426, upload-time = "2026-10-30T15:09:23.444Z" },
+ { url = "https://files.pythonhosted.org/packages/e6/21/1df51f94047dfb76c32fd7508036e4b55352d0f04fd0e3bee525a2122225/websockets-17.4-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ed7fe197b8bb66eef4f62c052e33752d456d2fdda74805469d4bd954950ab339", size = 228211, upload-time = "2026-10-30T15:09:24.902Z" },
+ { url = "https://files.pythonhosted.org/packages/cf/13/33d6a28471da4c30a98a2313c76f984be2056a703c5c532225dc28f5dba3/websockets-17.4-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:f0494d3d97502e270282cc6a1f8648cd7335b4fe8d5c452006e5b7f9d62b8892", size = 227106, upload-time = "2026-10-30T15:09:26.315Z" },
+ { url = "https://files.pythonhosted.org/packages/4a/86/ff8441e56ca677e51f9c826868fccde2a82bafe0ffa337fd38823b64253a/websockets-17.4-cp315-cp315-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ab9e4bd60796866c578e3afa0a17c98636887827a39c2e238a61989c18fd8e7b", size = 224499, upload-time = "2026-10-30T15:09:28.116Z" },
+ { url = "https://files.pythonhosted.org/packages/da/91/470d7a619ce4aa837ff5b6a5836227c43bdfdc42ef837cce48f508992438/websockets-17.4-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:8d148b4b295224d3a19e7bd9f02ebe7344de254b3b2c4d0628ce42ade0bca119", size = 227213, upload-time = "2026-10-30T15:09:29.847Z" },
+ { url = "https://files.pythonhosted.org/packages/a4/ba/55ed1422d6c73b5b81996ca195f2a256a339b26c553f5686afc2a926bc2b/websockets-17.4-cp315-cp315-musllinux_1_2_armv7l.whl", hash = "sha256:6597a1f7413189b4bc58437babcdf39b79da416e5c477ae98323a88ecc108188", size = 225876, upload-time = "2026-10-30T15:09:31.249Z" },
+ { url = "https://files.pythonhosted.org/packages/e9/59/f133d260e7a48d3e231c3a495eeb6114b880c1e96bfc781a949c1f73991c/websockets-17.4-cp315-cp315-musllinux_1_2_i686.whl", hash = "sha256:a9b0026538719f7a532998d84640ab2e91ca1a051ce4b8d924fdcb607307e921", size = 226403, upload-time = "2026-10-30T15:09:33.199Z" },
+ { url = "https://files.pythonhosted.org/packages/45/16/bb3355453ffdc31d3a14ddc0b50e058f13a97d2ed019ba6c380e03456169/websockets-17.4-cp315-cp315-musllinux_1_2_ppc64le.whl", hash = "sha256:a703f23ca15a51251fc94f1774683f5aa511588a7ad6bbb83960cdb2a4dcd978", size = 227558, upload-time = "2026-10-30T15:09:34.703Z" },
+ { url = "https://files.pythonhosted.org/packages/0c/ab/f3d55a83c2a4daa8bf62594b00a721dc8767518ccb833f932b0178eb6c19/websockets-17.4-cp315-cp315-musllinux_1_2_riscv64.whl", hash = "sha256:6b061ea7ca94966af7cd7a87cb189ff15972c103244611a2b1e8408c04f1dbe6", size = 225369, upload-time = "2026-10-30T15:09:36.204Z" },
+ { url = "https://files.pythonhosted.org/packages/88/66/3a320bfb718d9ae1eff15ba7790979c0e6fdec74e388430f426e6647d7f4/websockets-17.4-cp315-cp315-musllinux_1_2_s390x.whl", hash = "sha256:e6d2a185ec4ff0478089305abd833da3a9fda8803eb77325b89ebeae5585a3b7", size = 226268, upload-time = "2026-10-30T15:09:37.565Z" },
+ { url = "https://files.pythonhosted.org/packages/da/ee/1f9cdeeaaa2d5e158b7a40f971e74b641b35f977674b46229a5bb14e270a/websockets-17.4-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:9a4e95d56994a45ac76541853796720436ffe462957d0b1b89e87ff3d3b1627e", size = 226530, upload-time = "2026-10-30T15:09:39.171Z" },
+ { url = "https://files.pythonhosted.org/packages/da/28/506406ef09e6e168796f0187bdac0767f38a560e6785c4ee26e583aaaf76/websockets-17.4-cp315-cp315-win32.whl", hash = "sha256:b7ea61af293a153f2d67430038cf0efc10140caa623f2bdf5a9f5a457ff20f57", size = 218751, upload-time = "2026-10-30T15:09:40.748Z" },
+ { url = "https://files.pythonhosted.org/packages/25/a6/818647a9a9b3ee8bf131e9ab4fa1f2db9c6539f08e7084dd57549e0bd49d/websockets-17.4-cp315-cp315-win_amd64.whl", hash = "sha256:8b45d40f07081b377f0b837ddbfba4a9ab4a89a7b569d3c095436564e767a27b", size = 219045, upload-time = "2026-10-30T15:09:42.251Z" },
+ { url = "https://files.pythonhosted.org/packages/3d/b2/0338a9abec3df8556627af987352657ce1c8456f2060b54741570b174122/websockets-17.4-cp315-cp315-win_arm64.whl", hash = "sha256:29b615c2d5dd883f89c107a782681b73a7aa8b2bd894130310e45eb09b143913", size = 218957, upload-time = "2026-10-30T15:09:43.866Z" },
+ { url = "https://files.pythonhosted.org/packages/68/92/22e83d003cdbd22be5d35e28c260203e9fbb6c73b17a8e1509f4ad49e74d/websockets-17.4-cp315-cp315t-macosx_10_15_universal2.whl", hash = "sha256:c568e72f5425a74616daeb824afe28da460cbeea036432f41c6af5380b83300d", size = 219056, upload-time = "2026-10-30T15:09:45.528Z" },
+ { url = "https://files.pythonhosted.org/packages/2f/ca/071d38e7237f6ada4f577b74f7cbf5bc2ff7e1f9360ebd2837e74a22dfd4/websockets-17.4-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:bfc86dff091104eec770c29ea1831dbe191968a70aa2d8ae8eda3240d863970a", size = 216630, upload-time = "2026-10-30T15:09:46.951Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/76/22d4523f92d88b8f5ebfd3df7ab5e825146af100f7e2caeae7d359ae51ea/websockets-17.4-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:f5f06f59e19aa0770aaf651ce91c196fa620b35aed49a7e6c36e7335f3f83db7", size = 216841, upload-time = "2026-10-30T15:09:48.285Z" },
+ { url = "https://files.pythonhosted.org/packages/87/29/bc94be4a8bc19ef518e9efecd90a992f2991b7cfc6eb8a7430c4e2dc097c/websockets-17.4-cp315-cp315t-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:8f32a6cb0b101170abc2d8053af1473a2a29ef2a06d7130994ee8455a50f5dbf", size = 226992, upload-time = "2026-10-30T15:09:49.776Z" },
+ { url = "https://files.pythonhosted.org/packages/fa/25/58f69b5fec064764ddf5b88045397433cdb028f5b17c82fbe047fd31fff7/websockets-17.4-cp315-cp315t-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:72d0a731eccf482d5661c6711abfcd5921f3dbfbbb0789061e1918ef4bc89c70", size = 227258, upload-time = "2026-10-30T15:09:51.172Z" },
+ { url = "https://files.pythonhosted.org/packages/21/7c/bc932a54c5f1bd78461574aea539fc562fd2de494738a2c0675c4de860be/websockets-17.4-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:48c83103f82fce36191a9e3a8a80731d4340e2da544eae8c5aaaf6a45280e7d0", size = 228589, upload-time = "2026-10-30T15:09:52.628Z" },
+ { url = "https://files.pythonhosted.org/packages/c4/92/57ec83066d0bfa8a9b9f594bc61951073c4a86e34c562a5292604118dab6/websockets-17.4-cp315-cp315t-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c8052c3058c402d53602583da5ddcf1a15a63ab1efe0222f47c7be59944e6463", size = 230217, upload-time = "2026-10-30T15:09:54.058Z" },
+ { url = "https://files.pythonhosted.org/packages/6e/fc/0f2fe533b5b4dc22ac704b3357f5c99fa60a3837218b3c6c6ad63a60120e/websockets-17.4-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:0c22a9c6b2349204f56123dd0ca7779e989d68e336d0dc02ad823485d44925de", size = 229026, upload-time = "2026-10-30T15:09:55.727Z" },
+ { url = "https://files.pythonhosted.org/packages/7c/cc/ed28307835106a9a2dbd1a5efc597e8befa3d0a5f2b46a4c1e466b2f4759/websockets-17.4-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0dcce83837328e6795225ead920d3aa8d867cc9d6922aa558b0e9111599fdbb9", size = 227904, upload-time = "2026-10-30T15:09:57.221Z" },
+ { url = "https://files.pythonhosted.org/packages/39/4d/24ad7deddc7b31c3e430acc1c00c7b384ae42f1f973e9cc01aa42d19628f/websockets-17.4-cp315-cp315t-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:4b79ea31cf277c72d2307c6cb8bd32a2c151959e80a2f24fa85df620c5f2860c", size = 225253, upload-time = "2026-10-30T15:09:58.624Z" },
+ { url = "https://files.pythonhosted.org/packages/93/f6/9c92230ebfab42fee482b236c45c32ed34b24f7360080b400cce6cfcac11/websockets-17.4-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:3d27153a0413696767b98b5d96173a2a7e27cbfca02000a6069cea7b977765a7", size = 228089, upload-time = "2026-10-30T15:10:00.308Z" },
+ { url = "https://files.pythonhosted.org/packages/fe/7a/f523830e562da9b1d7a610174752753720c5e47e84b59b5fc47721d416bd/websockets-17.4-cp315-cp315t-musllinux_1_2_armv7l.whl", hash = "sha256:e0feb03a338f8f5e819ae23a73aabb7b46fd3b73f15f7df782dd12d99e2b02ad", size = 226743, upload-time = "2026-10-30T15:10:01.744Z" },
+ { url = "https://files.pythonhosted.org/packages/2c/e6/d1a56495cb7b1a0a4688145131be38cfd0c1342bde8125e136a6e6ee494e/websockets-17.4-cp315-cp315t-musllinux_1_2_i686.whl", hash = "sha256:9ff89e36451a19775e7556d95ebfa4a0b24b84cc87565d6b5d0ec8ea82bf3f2c", size = 227237, upload-time = "2026-10-30T15:10:03.417Z" },
+ { url = "https://files.pythonhosted.org/packages/44/41/c4e53f7ad63838d7eba6e82c5d743827e91dafe5e48c88a58f7e90043945/websockets-17.4-cp315-cp315t-musllinux_1_2_ppc64le.whl", hash = "sha256:ee6c4e0085bff5f6621bf9c53452584dcb91ebd993ce841e1f4061ac12b6b556", size = 228348, upload-time = "2026-10-30T15:10:05.149Z" },
+ { url = "https://files.pythonhosted.org/packages/ca/01/f6f78c3edeb8ad2b3c35a8c4117ec2cf9346215afaed8155af1e7933818b/websockets-17.4-cp315-cp315t-musllinux_1_2_riscv64.whl", hash = "sha256:1962f3bfde085af18878f1bb1aafb300c9a6ac33da886d0b0d97e33c9ce7a222", size = 226099, upload-time = "2026-10-30T15:10:06.588Z" },
+ { url = "https://files.pythonhosted.org/packages/8d/01/d8c11740231b63d1aba6bad6b1d4e72e8384b26c5448801da238ebe82c92/websockets-17.4-cp315-cp315t-musllinux_1_2_s390x.whl", hash = "sha256:42d7a6e818214336cf7af9a3aae5c6923f94f09fa07ac5aab98268ba2a98a6d5", size = 226956, upload-time = "2026-10-30T15:10:08.141Z" },
+ { url = "https://files.pythonhosted.org/packages/99/87/dbfd43ae01ea8fdb7478a1b3d58b32aca83bb9c4aa61ebe187d1e188ac8e/websockets-17.4-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:c9f8fdb7bbefc4907c2bb3593f810f2b67a2d5329e1d38724a1aa3d763cf63eb", size = 227253, upload-time = "2026-10-30T15:10:09.533Z" },
+ { url = "https://files.pythonhosted.org/packages/bf/8e/fbd968a75428063560ada02d564de8d6c49ca34ec29dfb78e5d900f4b0ef/websockets-17.4-cp315-cp315t-win32.whl", hash = "sha256:506623128d7a55b726a5a46e4c9a8eae4c9dd0f04ff60b041fe7df85b9d034fe", size = 218877, upload-time = "2026-10-30T15:10:10.925Z" },
+ { url = "https://files.pythonhosted.org/packages/f9/53/8f37a3943517bba77fc86f7c4162c455018c5b6d760c5e9c9a11de9e37f7/websockets-17.4-cp315-cp315t-win_amd64.whl", hash = "sha256:6e6558efbb256b532c9f06a21ba42bcac729718e365c505819f181f25e5c0eca", size = 219195, upload-time = "2026-10-30T15:10:12.336Z" },
+ { url = "https://files.pythonhosted.org/packages/69/35/cf33cba7248fcd80cfe5911ffe67aa2637f50c2bfaada123b1e9d2aeca90/websockets-17.4-cp315-cp315t-win_arm64.whl", hash = "sha256:fcac44d917672ad6257ad64b80b5168306f168a78cab5c0612e4c699caed18cf", size = 219117, upload-time = "2026-10-30T15:10:13.771Z" },
+ { url = "https://files.pythonhosted.org/packages/eb/27/9db924cb0dbf22eade78d98fbae77790f9bf6bc729e6ab29aef9d68e4156/websockets-17.4-py3-none-any.whl", hash = "sha256:c6bd2c9781915592b209160d0f017d4c7c82ad53075e6b0c24d2a89299e7f9e9", size = 212900, upload-time = "2026-10-30T15:10:34.482Z" },
+]
+
[[package]]
name = "werkzeug"
version = "3.1.9"