diff --git a/docs/table_transforms.md b/docs/table_transforms.md index 30523c22..119820b9 100644 --- a/docs/table_transforms.md +++ b/docs/table_transforms.md @@ -9,7 +9,7 @@ All six file-format ingestion pipelines support pluggable file pre-processing, r | [`jsonlines/pydantic.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/jsonlines/pydantic.py) | `jsonlines_pydantic` | `pyarrow.Table` | Keys of `ENTITY_MODELS` | | [`xsv/pipeline.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/xsv/pipeline.py) | `xsv` | `pyarrow.Table` | `--table-name` (or preset entrypoint `uniprot_idmapping`) | | [`xmltodict/pipeline.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py) | `xmltodict` | `list[dict]` (`pyarrow.Table` if schema map set) | `--table-name` (`settings.table_name`) | -| [`xmltopyarrow/pipeline.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py) | `xmltopyarrow` | `pyarrow.Table` | `--table-name` (or preset entrypoints `pubmed`, `uniprot`, `uniref50`, `uniref90`, `uniref100`) | +| [`xmltopyarrow/pipeline.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py) | `xmltopyarrow` | `pyarrow.Table` | `--table-name` (or preset entrypoints `pubmed`, `uniprot`, `uniref50`, `uniref90`, `uniref100`, `uniprot_cdm`, `uniref50_cdm`, `uniref90_cdm`, `uniref100_cdm`) | --- @@ -65,7 +65,7 @@ Create an importable Python module exporting `FILE_PREPROCESSORS`, `TABLE_TRANSF **`OUTPUT_SCHEMAS`** (`--output-schemas-module` / `CDL_OUTPUT_SCHEMAS_MODULE`): `dict[str, pa.Schema]` - - Optional mapping of `{destination_table_name: pyarrow.Schema}` overriding or supplying `pyarrow.Schema` definitions for transformed or split destination tables. Preset settings subclasses can also return `OutputTableHints` from `get_output_table_hints()` ([`src/cdm_data_loaders/core/transforms/hints.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/core/transforms/hints.py)) to register table/column descriptions and Iceberg partition specs on `DltResource` variants. + - Optional mapping of `{destination_table_name: pyarrow.Schema}` overriding or supplying `pyarrow.Schema` definitions for transformed or split destination tables. Preset settings subclasses can lock built-in transforms and output schemas off the CLI with `LockedTransformHooksMixin` ([`src/cdm_data_loaders/core/transforms/settings_mixin.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/core/transforms/settings_mixin.py)), attach table descriptions via `with_schema_description` ([`src/cdm_data_loaders/utils/arrow_schema.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/utils/arrow_schema.py)), and return `OutputTableHints` from `get_output_table_hints()` ([`src/cdm_data_loaders/core/transforms/hints.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/core/transforms/hints.py)) to register table/column descriptions and Iceberg partition specs on `DltResource` variants. --- @@ -488,23 +488,28 @@ import pyarrow.compute as pc from cdm_data_loaders.core.registry import TableTransformFn, TransformContext from cdm_data_loaders.core.transforms.routing import emit_raw_table - -RAW_ID_MAPPING_SCHEMA = pa.schema( - [ - pa.field("uniprot_id", pa.string(), nullable=False), - pa.field("db_reference_type", pa.string(), nullable=False), - pa.field("db_reference_id", pa.string(), nullable=False), - ], - metadata={"comment": "raw ID mapping data from UniProt"}, +from cdm_data_loaders.utils.arrow_schema import with_schema_description + +RAW_ID_MAPPING_SCHEMA = with_schema_description( + pa.schema( + [ + pa.field("uniprot_id", pa.string(), nullable=False), + pa.field("db_reference_type", pa.string(), nullable=False), + pa.field("db_reference_id", pa.string(), nullable=False), + ] + ), + "raw ID mapping data from UniProt", ) -ID_MAPPING_SCHEMA = pa.schema( - [ - pa.field("uniprot_id", pa.string(), nullable=False), - pa.field("db_reference", pa.string(), nullable=False), - pa.field("source", pa.string(), nullable=False), - ], - metadata={"comment": "Standardised mapping data from UniProt"}, +ID_MAPPING_SCHEMA = with_schema_description( + pa.schema( + [ + pa.field("uniprot_id", pa.string(), nullable=False), + pa.field("db_reference", pa.string(), nullable=False), + pa.field("source", pa.string(), nullable=False), + ] + ), + "Standardised mapping data from UniProt", ) diff --git a/docs_macros/tables/pipeline_tables.py b/docs_macros/tables/pipeline_tables.py index 3f04d3d0..617c87b7 100644 --- a/docs_macros/tables/pipeline_tables.py +++ b/docs_macros/tables/pipeline_tables.py @@ -8,6 +8,12 @@ from cdm_data_loaders.converters.sources.json_schema.strict_scalars import NON_EXTENDABLE_SCALAR_HINTS from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA from cdm_data_loaders.pipelines.presets.pubmed import PubMedXmlToPyArrowSettings +from cdm_data_loaders.pipelines.presets.uniprot_cdm import ( + UniProtSettings, + UniRef50CdmSettings, + UniRef90CdmSettings, + UniRef100CdmSettings, +) from cdm_data_loaders.pipelines.presets.uniprot_xml import ( UniProtXmlToPyArrowSettings, UniRef50XmlToPyArrowSettings, @@ -37,9 +43,13 @@ XMLTOPYARROW_SETTINGS_CLASSES: Final[tuple[type[XmlToPyArrowSettings], ...]] = ( XmlToPyArrowSettings, UniProtXmlToPyArrowSettings, + UniProtSettings, UniRef50XmlToPyArrowSettings, + UniRef50CdmSettings, UniRef90XmlToPyArrowSettings, + UniRef90CdmSettings, UniRef100XmlToPyArrowSettings, + UniRef100CdmSettings, PubMedXmlToPyArrowSettings, ) diff --git a/specs/pipelines/xmltopyarrow/presets/uniprot_cdm/01_cdm_ingest_and_rejected/case.yaml b/specs/pipelines/xmltopyarrow/presets/uniprot_cdm/01_cdm_ingest_and_rejected/case.yaml new file mode 100644 index 00000000..f4ad2631 --- /dev/null +++ b/specs/pipelines/xmltopyarrow/presets/uniprot_cdm/01_cdm_ingest_and_rejected/case.yaml @@ -0,0 +1,30 @@ +title: UniProtKB CDM Fan-Out and Missing-Accession Rejection (`_rejected`) +pipeline: uniprot_cdm +settings_file: settings.toml +rules: + - R-XML-DEFAULT-NS + - R-XML-TRANSFORMS + - R-XML-REPORT + - R-UNIPROT-CDM-PRESET +notes: | + A `` XML file with one complete `` (`P12345`) and one `` whose `` is blank. + `UniProtSettings` fans the valid entry across `entity`, `identifier`, `name`, `protein`, and + `entity_x_publication` (`UNIPROT_CDM_SCHEMA_MAP`) and routes the blank-accession entry to `_rejected` + with `table_name="entry"` and `source_file="uniprot_sample.xml"`. +runs: + - id: default + input_dir: input + expected_report: + status: completed + ingested_files: + - uniprot_sample.xml + pipeline_load_info_errs: null + expected_counts: + entity: 1 + identifier: 4 + name: 3 + protein: 1 + entity_x_publication: 1 + _rejected: 1 + expected_tables: null + expected_json: null diff --git a/specs/pipelines/xmltopyarrow/presets/uniprot_cdm/01_cdm_ingest_and_rejected/input/uniprot_sample.xml b/specs/pipelines/xmltopyarrow/presets/uniprot_cdm/01_cdm_ingest_and_rejected/input/uniprot_sample.xml new file mode 100644 index 00000000..be1e88e2 --- /dev/null +++ b/specs/pipelines/xmltopyarrow/presets/uniprot_cdm/01_cdm_ingest_and_rejected/input/uniprot_sample.xml @@ -0,0 +1,52 @@ + + + + P12345 + Q67890 + TEST_ENTRY + + + Test protein + + + + testGene + + + Test organism + + + + + A test paper. + + + + + + SEQUENCE + + + A test function. + + + + + ACGTA + + + + MISSING_ACC + + + Unindexed entry + + + + Test organism + + + + ACGT + + diff --git a/specs/pipelines/xmltopyarrow/presets/uniprot_cdm/01_cdm_ingest_and_rejected/settings.toml b/specs/pipelines/xmltopyarrow/presets/uniprot_cdm/01_cdm_ingest_and_rejected/settings.toml new file mode 100644 index 00000000..cad4909a --- /dev/null +++ b/specs/pipelines/xmltopyarrow/presets/uniprot_cdm/01_cdm_ingest_and_rejected/settings.toml @@ -0,0 +1,2 @@ +loader_file_format = "parquet" +use_output_dir_for_pipeline_metadata = true diff --git a/specs/pipelines/xmltopyarrow/presets/uniref50_cdm/01_cdm_ingest_and_rejected/case.yaml b/specs/pipelines/xmltopyarrow/presets/uniref50_cdm/01_cdm_ingest_and_rejected/case.yaml new file mode 100644 index 00000000..bbdb5928 --- /dev/null +++ b/specs/pipelines/xmltopyarrow/presets/uniref50_cdm/01_cdm_ingest_and_rejected/case.yaml @@ -0,0 +1,28 @@ +title: UniRef50 CDM Cluster Fan-Out and Missing-ID Rejection (`_rejected`) +pipeline: uniref50_cdm +settings_file: settings.toml +rules: + - R-XML-DEFAULT-NS + - R-XML-TRANSFORMS + - R-XML-REPORT + - R-UNIREF-CDM-PRESET +notes: | + A `` XML file with one valid `` (`UniRef50_A0AB34IYJ6`) and one `` missing `id`. + `UniRef50CdmSettings` fans the valid cluster across `entity`, `cluster`, and `clustermember` + (`UNIREF_CDM_SCHEMA_MAP`) and routes the entry without `id` to `_rejected` with `table_name="entry"` + and `source_file="uniref50_sample.xml"`. +runs: + - id: default + input_dir: input + expected_report: + status: completed + ingested_files: + - uniref50_sample.xml + pipeline_load_info_errs: null + expected_counts: + entity: 1 + cluster: 1 + clustermember: 2 + _rejected: 1 + expected_tables: null + expected_json: null diff --git a/specs/pipelines/xmltopyarrow/presets/uniref50_cdm/01_cdm_ingest_and_rejected/input/uniref50_sample.xml b/specs/pipelines/xmltopyarrow/presets/uniref50_cdm/01_cdm_ingest_and_rejected/input/uniref50_sample.xml new file mode 100644 index 00000000..99057447 --- /dev/null +++ b/specs/pipelines/xmltopyarrow/presets/uniref50_cdm/01_cdm_ingest_and_rejected/input/uniref50_sample.xml @@ -0,0 +1,23 @@ + + + + Cluster: PKZILLA-1 giant polyketide synthase + + + + + + + ACGT + + + + + + + Cluster: Missing ID + + + + + diff --git a/specs/pipelines/xmltopyarrow/presets/uniref50_cdm/01_cdm_ingest_and_rejected/settings.toml b/specs/pipelines/xmltopyarrow/presets/uniref50_cdm/01_cdm_ingest_and_rejected/settings.toml new file mode 100644 index 00000000..cad4909a --- /dev/null +++ b/specs/pipelines/xmltopyarrow/presets/uniref50_cdm/01_cdm_ingest_and_rejected/settings.toml @@ -0,0 +1,2 @@ +loader_file_format = "parquet" +use_output_dir_for_pipeline_metadata = true diff --git a/specs/pipelines/xmltopyarrow/rules.yaml b/specs/pipelines/xmltopyarrow/rules.yaml index 62dae287..e20e02c7 100644 --- a/specs/pipelines/xmltopyarrow/rules.yaml +++ b/specs/pipelines/xmltopyarrow/rules.yaml @@ -21,7 +21,7 @@ intro: | 1. **Target selection, preset configurations, and PyArrow schema resolution** (`XmlToPyArrowSettings`, `PresetXmlToPyArrowSettings`, `LevelPresetXmlToPyArrowSettings`). 2. **Streaming XML element extraction, default namespace resolution, and XSD-driven PyArrow table coercion** (`process_xml_file_to_pyarrow`). 3. **File discovery, `.gz` decompression, MD5 sidecar pre-processing, dead-letter routing (`_rejected`), table transforms, and pipeline reporting** (`run_xmltopyarrow_pipeline`). - 4. **Preset subclass behavior** (`pubmed`, `uniprot`, `uniref50`, `uniref90`, `uniref100`). + 4. **Preset subclass behavior** (`pubmed`, `uniprot`, `uniprot_cdm`, `uniref50`, `uniref50_cdm`, `uniref90`, `uniref90_cdm`, `uniref100`, `uniref100_cdm`). 5. **Errors** (`XmlToPyArrowConfigError` subclasses) raised for invalid target selection, invalid XSDs, or invalid pipeline hooks. sections: @@ -103,10 +103,11 @@ sections: path), `line_no=None`, `raw_record=None`, and `error_detail`. - id: R-XML-TRANSFORMS statement: | - When `transforms_module` defines a `RecordTransformFn` for a target table, converted XML element dicts - are passed to the transform before buffering. The transform may drop records (`{}`), modify fields, or - fan out records across multiple destination tables, coercing each table with `OUTPUT_SCHEMAS` when - provided. + When `transforms_module` defines a `RecordTransformFn` for a target table, `route_and_coerce_record` + passes converted XML element dicts (and `TransformContext(table_name, source_file)` when the callable + declares a context parameter) to the transform before buffering. The transform may drop records (`{}`), + route rejected records to `_rejected` (`REJECTED_PYARROW_SCHEMA`) with `table_name` and `source_file` + provenance, update fields, or fan out records across destination tables coerced with `OUTPUT_SCHEMAS`. - id: R-XML-REPORT statement: | Every pipeline execution writes one audit row to `_load_info` and a YAML summary to @@ -132,3 +133,23 @@ sections: When a `.xml.gz` file fails `validate_md5_sidecar` in `pubmed`, the file is omitted from `ingested_files`, a file-level provenance row (`table_name=None`, `line_no=None`, `raw_record=None`) is written to `_rejected`, and `report.txt` records `status: completed with errors` and `load_errors`. + + - id: uniprot_cdm_presets + title: UniProtKB and UniRef CDM Preset Subclasses + rules: + - id: R-UNIPROT-CDM-PRESET + statement: | + `UniProtSettings` (`uniprot_cdm` CLI) subclasses `LockedTransformHooksMixin` and + `PresetXmlToPyArrowSettings`, locking `dataset_name="uniprot_cdm"`, `table_name="entry"`, + `xml_tag="entry"`, `transforms_module=None`, and `output_schemas_module=None`, and binding + `make_uniprot_cdm_transform` and `UNIPROT_CDM_SCHEMA_MAP` (`entity`, `identifier`, `name`, + `protein`, `entity_x_publication`). Entries missing `` or raising parse errors route + to `_rejected` with `table_name="entry"` and `source_file` provenance. + - id: R-UNIREF-CDM-PRESET + statement: | + `UniRef50CdmSettings`, `UniRef90CdmSettings`, and `UniRef100CdmSettings` (`uniref50_cdm`, + `uniref90_cdm`, `uniref100_cdm` CLIs) subclass `UniRefCdmSettings` (`LockedTransformHooksMixin` + and `PresetXmlToPyArrowSettings`), locking `dataset_name`, `table_name="entry"`, `xml_tag="entry"`, + `transforms_module=None`, and `output_schemas_module=None`, and binding `make_uniref_cdm_transform` + and `UNIREF_CDM_SCHEMA_MAP` (`entity`, `cluster`, `clustermember`). Entries missing `_id` or raising + parse errors route to `_rejected` with `table_name="entry"` and `source_file` provenance. diff --git a/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/case.yaml b/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/case.yaml index 1f5a723e..58b99b51 100644 --- a/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/case.yaml +++ b/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/case.yaml @@ -7,12 +7,13 @@ rules: notes: | - `transforms_module="specs.pipelines.xmltopyarrow.transforms"` defines `TABLE_TRANSFORMS["measurements"] = transform_measurement` and `OUTPUT_SCHEMAS` for `measurements` and `measurement_tags`. - `XmlToPyArrowSettings` skips default XSD-to-PyArrow schema generation for `measurements` and uses `OUTPUT_SCHEMAS`. - - `transform_measurement` drops `` (`s2`), uppercases `_id` into `sensor_id`, computes `scaled_value`, and fans out `` values into `measurement_tags`. + - `transform_measurement` receives `TransformContext(table_name="measurements", source_file="data.xml")`, drops `` (`s2`), routes `` (`s4`) to `_rejected` with `table_name` and `source_file` provenance, uppercases `_id` into `sensor_id`, computes `scaled_value`, and fans out `` values into `measurement_tags`. runs: - id: default data: data.xml expected_json: expected.json expected_table: expected_table.py + expected_rejected: expected_rejected.json extra_expected_json: measurement_tags: expected_tags.json params: diff --git a/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/data.xml b/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/data.xml index d555bc64..0ce3c1d6 100644 --- a/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/data.xml +++ b/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/data.xml @@ -12,5 +12,8 @@ 3.5 + + -1.0 + diff --git a/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/expected_rejected.json b/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/expected_rejected.json new file mode 100644 index 00000000..693f654d --- /dev/null +++ b/specs/pipelines/xmltopyarrow/spec/06_table_transforms_and_output_schemas/expected_rejected.json @@ -0,0 +1,9 @@ +[ + { + "error_detail": "Invalid measurement status", + "line_no": null, + "raw_record": "{\"_id\": \"s4\", \"_status\": \"invalid\", \"value\": -1.0}", + "source_file": "data.xml", + "table_name": "measurements" + } +] diff --git a/specs/pipelines/xmltopyarrow/transforms.py b/specs/pipelines/xmltopyarrow/transforms.py index 63cc8ee0..3c5af982 100644 --- a/specs/pipelines/xmltopyarrow/transforms.py +++ b/specs/pipelines/xmltopyarrow/transforms.py @@ -4,8 +4,9 @@ import pyarrow as pa +from cdm_data_loaders.core.buffer import make_rejected_table_result from cdm_data_loaders.core.md5_preprocessor import validate_md5_sidecar -from cdm_data_loaders.core.registry import FilePreprocessFn, RecordTransformFn +from cdm_data_loaders.core.registry import FilePreprocessFn, RecordTransformFn, TransformContext FILE_PREPROCESSORS: tuple[FilePreprocessFn, ...] = (validate_md5_sidecar,) @@ -25,10 +26,17 @@ ) -def transform_measurement(row: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: - """Drop draft measurements, compute ``scaled_value``, and fan out tags to ``measurement_tags``.""" +def transform_measurement(row: dict[str, Any], ctx: TransformContext) -> dict[str, list[dict[str, Any]]]: + """Drop draft rows, reject invalid rows with ``ctx`` provenance, compute ``scaled_value``, and fan out tags.""" if row.get("_status") == "drop": return {} + if row.get("_status") == "invalid": + return make_rejected_table_result( + ctx.table_name, + row, + "Invalid measurement status", + source_file=ctx.source_file, + ) sensor_id = str(row["_id"]).upper() value = float(row["value"]) routed: dict[str, list[dict[str, Any]]] = { diff --git a/src/cdm_data_loaders/cli/ncbi_ftp_filter_updates.py b/src/cdm_data_loaders/cli/ncbi_ftp_filter_updates.py index 52556f6a..0ba25d14 100644 --- a/src/cdm_data_loaders/cli/ncbi_ftp_filter_updates.py +++ b/src/cdm_data_loaders/cli/ncbi_ftp_filter_updates.py @@ -10,9 +10,9 @@ from cdm_data_loaders.core.cli import CliRunner from cdm_data_loaders.core.settings import LoggerSettings, default_settings_with_shortcuts +from cdm_data_loaders.datasets.ncbi_ftp.constants import Source from cdm_data_loaders.datasets.ncbi_ftp.updates import ( BATCH_SIZE, - Source, Splits, build_splits, compute_batch_width, diff --git a/src/cdm_data_loaders/converters/facades/jsonschema_to_pyarrow.py b/src/cdm_data_loaders/converters/facades/jsonschema_to_pyarrow.py index c9b83282..8f0989b2 100644 --- a/src/cdm_data_loaders/converters/facades/jsonschema_to_pyarrow.py +++ b/src/cdm_data_loaders/converters/facades/jsonschema_to_pyarrow.py @@ -72,7 +72,6 @@ def _reader(self) -> StrictJsonSchemaReader: :rtype: StrictJsonSchemaReader """ forbidden_formats = frozenset(fmt for fmt, dtype in self.format_map.items() if is_non_json_arrow_type(dtype)) - forbidden_formats = frozenset(fmt for fmt, dtype in self.format_map.items() if is_non_json_arrow_type(dtype)) return StrictJsonSchemaReader( self.extension_registry, error_type=self._error_cls, diff --git a/src/cdm_data_loaders/core/buffer.py b/src/cdm_data_loaders/core/buffer.py index 5cba9d8d..8c9c3c2a 100644 --- a/src/cdm_data_loaders/core/buffer.py +++ b/src/cdm_data_loaders/core/buffer.py @@ -119,6 +119,46 @@ def make_rejected_table_result( } +def resolve_destination_schema( + table_name: str, + schema_map: ArrowSchemaRegistry | None = None, +) -> pa.Schema | None: + """Resolve the destination ``pyarrow.Schema`` for ``table_name``, including ``_rejected``. + + :param table_name: destination table name + :type table_name: str + :param schema_map: active ``ArrowSchemaRegistry`` for pipeline output tables, if any + :type schema_map: ArrowSchemaRegistry | None + :return: ``REJECTED_PYARROW_SCHEMA`` for ``_rejected``, registered table schema when mapped, else ``None`` + :rtype: pa.Schema | None + """ + if table_name == REJECTED_TABLE_NAME: + return REJECTED_PYARROW_SCHEMA + if schema_map is not None: + return schema_map.registry.get(table_name) + return None + + +def format_rejected_records( + rows: list[dict[str, Any]], + *, + use_pyarrow: bool = True, +) -> DataItemWithMeta: + """Format ``_rejected`` row dictionaries and tag the page with ``_rejected`` table metadata. + + :param rows: standardized ``_rejected`` row dictionaries + :type rows: list[dict[str, Any]] + :param use_pyarrow: when ``True``, convert ``rows`` using ``REJECTED_PYARROW_SCHEMA`` + :type use_pyarrow: bool + :return: ``DataItemWithMeta`` carrying a ``pyarrow.Table`` or ``rows`` for ``_rejected`` + :rtype: DataItemWithMeta + """ + page: pa.Table | list[dict[str, Any]] = ( + pa.Table.from_pylist(rows, schema=REJECTED_PYARROW_SCHEMA) if use_pyarrow else rows + ) + return dlt.mark.with_table_name(page, REJECTED_TABLE_NAME) + + class DictBuffer(BaseModel): """Accumulates rows per table and yields them as pages to dlt. @@ -158,10 +198,9 @@ def _make_page(self, table_name: str, rows: list[Any]) -> pa.Table | list[Any]: """ if not self.use_pyarrow: return rows - if table_name == REJECTED_TABLE_NAME: - return pa.Table.from_pylist(rows, schema=REJECTED_PYARROW_SCHEMA) - if self.schema_map and table_name in self.schema_map.registry: - return pa.Table.from_pylist(rows, schema=self.schema_map.registry[table_name]) + target_schema = resolve_destination_schema(table_name, self.schema_map) + if target_schema is not None: + return pa.Table.from_pylist(rows, schema=target_schema) return rows def _flush_if_full(self, table_name: str) -> Generator[DataItemWithMeta, Any]: diff --git a/src/cdm_data_loaders/core/filesystem.py b/src/cdm_data_loaders/core/filesystem.py index 24fb14bf..f736a2a7 100644 --- a/src/cdm_data_loaders/core/filesystem.py +++ b/src/cdm_data_loaders/core/filesystem.py @@ -1,12 +1,11 @@ """Pipeline filesystem input set up and processing.""" -from collections.abc import Callable, Generator, Iterable +from collections.abc import Callable, Generator, Iterable, Mapping from dataclasses import dataclass from logging import Logger, getLogger from typing import TYPE_CHECKING, Any import dlt -import pyarrow as pa from dlt.common.schema.typing import TSchemaContractDict from dlt.common.storages.fsspec_filesystem import FileItemDict from dlt.common.typing import TDataItems @@ -14,8 +13,8 @@ from dlt.sources.filesystem import filesystem from dlt.sources.filesystem.helpers import fsspec_from_resource -from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA, make_rejected_row -from cdm_data_loaders.core.fields import ICEBERG, PARQUET, REJECTED_TABLE_NAME, LoaderFileFormatEnum +from cdm_data_loaders.core.buffer import format_rejected_records, make_rejected_row +from cdm_data_loaders.core.fields import ICEBERG, PARQUET, LoaderFileFormatEnum from cdm_data_loaders.core.file_items import source_id from cdm_data_loaders.core.settings import ( FileFormatIngestSettings, @@ -233,14 +232,9 @@ def _preprocess_and_read( str(exc), source_file=src_id, ) - rejected_page: TDataItems = ( - pa.Table.from_pylist([rejected_row], schema=REJECTED_PYARROW_SCHEMA) - if getattr(settings, "use_pyarrow", True) - else [rejected_row] - ) - yield dlt.mark.with_table_name( - rejected_page, - REJECTED_TABLE_NAME, + yield format_rejected_records( + [rejected_row], + use_pyarrow=getattr(settings, "use_pyarrow", True), ) failed = True break @@ -326,6 +320,22 @@ def build_per_file_filesystem_resource( ) +def resolve_pipeline_name(settings: object, default_pipeline_name: str) -> str: + """Resolve the active pipeline name from ``settings.model_config['cli_prog_name']`` or ``default_pipeline_name``. + + :param settings: pipeline settings instance + :type settings: object + :param default_pipeline_name: fallback pipeline name when ``cli_prog_name`` is unset or empty + :type default_pipeline_name: str + :return: resolved pipeline name + :rtype: str + """ + model_config = getattr(settings, "model_config", None) + if isinstance(model_config, Mapping) and (prog_name := model_config.get("cli_prog_name")): + return str(prog_name) + return default_pipeline_name + + def build_file_ingest_kwargs( settings: FileFormatIngestSettings, pipeline_name: str, @@ -336,14 +346,15 @@ def build_file_ingest_kwargs( :param settings: file-format ingest settings with ``dataset_name`` and ``loader_file_format`` :type settings: FileFormatIngestSettings - :param pipeline_name: name of the dlt pipeline + :param pipeline_name: default dlt pipeline name when ``settings.model_config['cli_prog_name']`` is unset :type pipeline_name: str - :param fallback_dataset_name: whether to fall back to ``pipeline_name`` when ``settings.dataset_name`` is None + :param fallback_dataset_name: whether to fall back to the resolved pipeline name when ``settings.dataset_name`` is None :type fallback_dataset_name: bool :return: mapping with ``pipeline_kwargs`` and ``pipeline_run_kwargs`` suitable for ``run_pipeline`` :rtype: dict[str, dict[str, Any]] """ - dataset_name = (settings.dataset_name or pipeline_name) if fallback_dataset_name else settings.dataset_name + resolved_pipeline_name = resolve_pipeline_name(settings, pipeline_name) + dataset_name = (settings.dataset_name or resolved_pipeline_name) if fallback_dataset_name else settings.dataset_name run_kwargs: dict[str, Any] = ( {"loader_file_format": str(PARQUET), "table_format": str(ICEBERG)} if settings.loader_file_format == LoaderFileFormatEnum.ICEBERG @@ -351,7 +362,7 @@ def build_file_ingest_kwargs( ) return { "pipeline_kwargs": { - "pipeline_name": pipeline_name, + "pipeline_name": resolved_pipeline_name, "dataset_name": dataset_name, }, "pipeline_run_kwargs": run_kwargs, diff --git a/src/cdm_data_loaders/core/transforms/hints.py b/src/cdm_data_loaders/core/transforms/hints.py index ad246c95..5e5ce083 100644 --- a/src/cdm_data_loaders/core/transforms/hints.py +++ b/src/cdm_data_loaders/core/transforms/hints.py @@ -13,6 +13,7 @@ from pydantic import BaseModel, ConfigDict, InstanceOf from cdm_data_loaders.core.registry import RegistryKey +from cdm_data_loaders.utils.arrow_schema import SCHEMA_COMMENT_METADATA_KEY if TYPE_CHECKING: from dlt.common.schema.typing import TAnySchemaColumns @@ -27,15 +28,15 @@ def extract_schema_descriptions(schema: pa.Schema) -> tuple[str | None, dict[str :rtype: tuple[str | None, dict[str, str]] """ table_desc: str | None = None - if schema.metadata and b"comment" in schema.metadata: - decoded = schema.metadata[b"comment"].decode("utf-8") + if schema.metadata and SCHEMA_COMMENT_METADATA_KEY in schema.metadata: + decoded = schema.metadata[SCHEMA_COMMENT_METADATA_KEY].decode("utf-8") if decoded: table_desc = decoded col_descs: dict[str, str] = {} for field in schema: - if field.metadata and b"comment" in field.metadata: - col_comment = field.metadata[b"comment"].decode("utf-8") + if field.metadata and SCHEMA_COMMENT_METADATA_KEY in field.metadata: + col_comment = field.metadata[SCHEMA_COMMENT_METADATA_KEY].decode("utf-8") if col_comment: col_descs[field.name] = col_comment return table_desc, col_descs diff --git a/src/cdm_data_loaders/core/transforms/routing.py b/src/cdm_data_loaders/core/transforms/routing.py index 99826654..5630a18c 100644 --- a/src/cdm_data_loaders/core/transforms/routing.py +++ b/src/cdm_data_loaders/core/transforms/routing.py @@ -8,8 +8,7 @@ from pydantic import BaseModel from cdm_data_loaders.coercion.json_to_arrow import coerce_json_record_for_pyarrow -from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA -from cdm_data_loaders.core.fields import REJECTED_TABLE_NAME +from cdm_data_loaders.core.buffer import resolve_destination_schema from cdm_data_loaders.core.registry import ( ArrowSchemaRegistry, TableTransformFn, @@ -362,7 +361,7 @@ def _route_record( result: dict[str, list[dict[str, Any]]] = {} for out_table, out_val in routed.items(): - target_schema = schema_map.registry.get(out_table) if schema_map is not None else None + target_schema = resolve_destination_schema(out_table, schema_map) if schema_map is not None else None if isinstance(out_val, pa.Table): cast_tbl = align_and_cast_table(out_val, target_schema) if target_schema is not None else out_val rows = cast_tbl.to_pylist() @@ -449,11 +448,7 @@ def route_and_coerce_table( result[out_table] = rows continue - target_schema = ( - REJECTED_PYARROW_SCHEMA - if out_table == REJECTED_TABLE_NAME - else (schema_map.registry.get(out_table) if schema_map is not None else None) - ) + target_schema = resolve_destination_schema(out_table, schema_map) if isinstance(out_val, pa.Table): result[out_table] = align_and_cast_table(out_val, target_schema) if target_schema is not None else out_val elif target_schema is not None: diff --git a/src/cdm_data_loaders/core/transforms/settings_mixin.py b/src/cdm_data_loaders/core/transforms/settings_mixin.py index 685e20f8..cbe26dfd 100644 --- a/src/cdm_data_loaders/core/transforms/settings_mixin.py +++ b/src/cdm_data_loaders/core/transforms/settings_mixin.py @@ -1,11 +1,10 @@ """Settings mixin and table validation helpers for pluggable pre-processors, transforms, and schemas.""" -from collections.abc import Callable, Iterable +from collections.abc import Callable, Iterable, Mapping from typing import Any, ClassVar, Self import pyarrow as pa -from pydantic import PrivateAttr, model_validator -from pydantic_settings import BaseSettings +from pydantic import BaseModel, PrivateAttr, model_validator from cdm_data_loaders.core.fields import ( PYARROW_FILE_FORMATS, @@ -29,6 +28,13 @@ from cdm_data_loaders.core.transforms.routing import compose_transforms, resolve_schema_map +class LockedTransformHooksMixin: + """Mixin locking ``transforms_module`` and ``output_schemas_module`` as ``None`` class variables.""" + + transforms_module: ClassVar[str | None] = None + output_schemas_module: ClassVar[str | None] = None + + def validate_table_transforms( settings: object, valid_tables: Iterable[str], @@ -86,7 +92,47 @@ def needs_default_pyarrow_schema( return not (output_schemas is not None and table_name in output_schemas.registry) -class TransformSettingsMixin(BaseSettings): +def reconcile_multi_table_schema_map( + source_tables: Iterable[str], + build_default_schema: Callable[[str], pa.Schema | None], + *, + use_pyarrow: bool = True, + table_transforms: TransformRegistry | None = None, + output_schemas: ArrowSchemaRegistry | None = None, +) -> ArrowSchemaRegistry | None: + """Reconcile explicit ``output_schemas`` with per-table default PyArrow schemas. + + :param source_tables: input table names eligible for default PyArrow schema generation + :type source_tables: Iterable[str] + :param build_default_schema: callable returning the default ``pa.Schema`` (or ``None``) for a table name + :type build_default_schema: Callable[[str], pa.Schema | None] + :param use_pyarrow: whether PyArrow output is enabled + :type use_pyarrow: bool + :param table_transforms: optional configured table transforms registry + :type table_transforms: TransformRegistry | None + :param output_schemas: optional explicit output schemas registry + :type output_schemas: ArrowSchemaRegistry | None + :return: merged ``ArrowSchemaRegistry`` or ``None`` when unschematized or ``use_pyarrow`` is ``False`` + :rtype: ArrowSchemaRegistry | None + """ + if not use_pyarrow: + return None + resolved: dict[str, pa.Schema] = dict(output_schemas.registry) if output_schemas is not None else {} + for table_name in source_tables: + if not needs_default_pyarrow_schema( + table_name, + use_pyarrow=use_pyarrow, + table_transforms=table_transforms, + output_schemas=output_schemas, + ): + continue + default_schema = build_default_schema(table_name) + if default_schema is not None: + resolved[table_name] = default_schema + return ArrowSchemaRegistry(registry=resolved) if resolved else None + + +class TransformSettingsMixin(BaseModel): """Mixin for pipelines that accept ``FILE_PREPROCESSORS``, ``TABLE_TRANSFORMS``, and ``OUTPUT_SCHEMAS`` modules.""" DEFAULT_FILE_PREPROCESSORS: ClassVar[tuple[FilePreprocessFn, ...]] = () @@ -201,6 +247,22 @@ def _resolve_default_table_schema( return default_pyarrow_schema return build_default() + def _resolve_multi_table_schema_map(self, default_schemas: Mapping[str, pa.Schema]) -> None: + """Populate ``_schema_map`` from ``_unreconciled_output_schemas`` and ``default_schemas``. + + :param default_schemas: mapping of input table names to default ``pa.Schema`` instances + :type default_schemas: Mapping[str, pa.Schema] + :return: ``None`` + :rtype: None + """ + self._schema_map = reconcile_multi_table_schema_map( + default_schemas, + default_schemas.get, + use_pyarrow=self.use_pyarrow, + table_transforms=self._table_transforms, + output_schemas=self._unreconciled_output_schemas, + ) + def _resolve_settings_schema_map(self, default_pyarrow_schema: pa.Schema | None = None) -> None: """Resolve ``_schema_map`` from ``_unreconciled_output_schemas``, ``_table_transforms``, and ``default_pyarrow_schema``. diff --git a/src/cdm_data_loaders/datasets/all_the_bacteria/osf.py b/src/cdm_data_loaders/datasets/all_the_bacteria/osf.py index 6549bfad..18e59316 100644 --- a/src/cdm_data_loaders/datasets/all_the_bacteria/osf.py +++ b/src/cdm_data_loaders/datasets/all_the_bacteria/osf.py @@ -7,9 +7,7 @@ from pathlib import Path from typing import Any, Final -import requests - -from cdm_data_loaders.io.sync_client import FileDownloader +from cdm_data_loaders.io.sync_client import FileDownloader, fetch_json logger: Logger = getLogger(__name__) @@ -64,10 +62,11 @@ def download_osf_index_tsv(raw_data_dir: Path) -> Path: raw_data_dir.mkdir(parents=True, exist_ok=True) api_url = f"https://api.osf.io/v2/files/{ALL_FILES_TSV_FILE_ID}/" - resp = requests.get(api_url, headers={"accept": "application/json"}, timeout=OSF_TIMEOUT_SECONDS) - resp.raise_for_status() - - resp_json = resp.json() + resp_json = fetch_json( + api_url, + timeout=OSF_TIMEOUT_SECONDS, + extra_headers={"accept": "application/json"}, + ) all_files_tsv_download = resp_json.get("data", {}).get("links", {}).get("download") if all_files_tsv_download is None: logger.error("Could not find download URL in OSF API response:") diff --git a/src/cdm_data_loaders/datasets/ncbi_ftp/assembly.py b/src/cdm_data_loaders/datasets/ncbi_ftp/assembly.py index b9b6ea54..ce4066f9 100644 --- a/src/cdm_data_loaders/datasets/ncbi_ftp/assembly.py +++ b/src/cdm_data_loaders/datasets/ncbi_ftp/assembly.py @@ -1,5 +1,6 @@ """NCBI FTP assembly path helpers, file filters, and single-assembly FTP download.""" +import contextlib import time from dataclasses import dataclass from ftplib import FTP @@ -15,7 +16,7 @@ ftp_retrieve_text, ftp_session, ) -from cdm_data_loaders.utils.checksums import compute_file_checksum, parse_md5_checksums_file +from cdm_data_loaders.utils.checksums import parse_md5_checksums_file logger: Logger = getLogger(__name__) @@ -44,6 +45,17 @@ class _AssemblyDownloadContext: stats: dict[str, Any] +def matches_assembly_file_filter(path: PurePosixPath) -> bool: + """Return ``True`` when ``path`` ends with one of the suffixes in ``FILE_FILTERS``. + + :param path: relative or remote file path + :type path: PurePosixPath + :return: ``True`` if ``path`` matches ``FILE_FILTERS`` + :rtype: bool + """ + return any(path.match(f"**{suffix}") for suffix in FILE_FILTERS) + + def filter_assembly_checksums(checksums: dict[PurePosixPath, str]) -> dict[PurePosixPath, str]: """Filter a parsed ``md5checksums.txt`` mapping to files matching ``FILE_FILTERS``. @@ -52,7 +64,7 @@ def filter_assembly_checksums(checksums: dict[PurePosixPath, str]) -> dict[PureP :return: filtered mapping retaining entries whose suffixes match ``FILE_FILTERS`` :rtype: dict[PurePosixPath, str] """ - return {fname: md5 for fname, md5 in checksums.items() if any(fname.match(f"**{s}") for s in FILE_FILTERS)} + return {fname: md5 for fname, md5 in checksums.items() if matches_assembly_file_filter(fname)} def build_accession_path(assembly_dir: PurePosixPath) -> PurePosixPath: @@ -64,8 +76,8 @@ def build_accession_path(assembly_dir: PurePosixPath) -> PurePosixPath: :rtype: PurePosixPath :raises ValueError: if ``assembly_dir`` does not start with a valid RefSeq or GenBank accession """ - if m := ACCESSION_PARTS_REGEX.match(f"{assembly_dir}"): - return PurePosixPath("raw_data", *m.groups(), assembly_dir) + if m := ACCESSION_PARTS_REGEX.match(str(assembly_dir)): + return PurePosixPath("raw_data", m.group("db"), m.group("p1"), m.group("p2"), m.group("p3"), assembly_dir) msg = f"Cannot parse accession: {assembly_dir}" raise ValueError(msg) @@ -80,26 +92,48 @@ def parse_assembly_path(assembly_path: PurePosixPath) -> tuple[str, PurePosixPat :raises ValueError: if ``assembly_path`` cannot be parsed """ if m := ASSEMBLY_PATH_REGEX.search(str(assembly_path)): - return m.group(3), PurePosixPath(m.group(1)), m.group(2) + return m.group("db"), PurePosixPath(m.group("assembly_dir")), m.group("accession") msg = f"Cannot parse assembly path: {assembly_path}" raise ValueError(msg) +def extract_accession(entry: str | PurePosixPath) -> str: + """Extract the exact versioned accession ID from an FTP path, S3 key, directory name, or bare accession. + + :param entry: path or accession token (for example, ``"/genomes/all/.../GCF_000001215.4_Release_6/"`` or ``"GCF_000001215.4"``) + :type entry: str | PurePosixPath + :return: exact accession string (for example, ``"GCF_000001215.4"``) + :rtype: str + :raises ValueError: if ``entry`` does not contain a valid RefSeq or GenBank accession + """ + raw = str(entry).strip() + if not raw: + msg = f"Cannot parse accession: {entry}" + raise ValueError(msg) + with contextlib.suppress(ValueError): + _db, _assembly_dir, accession = parse_assembly_path(PurePosixPath(raw)) + return accession + if m := ACCESSION_PARTS_REGEX.match(raw.strip("/")): + return m.group("accession") + msg = f"Cannot parse accession: {entry}" + raise ValueError(msg) + + def _download_and_verify( ftp: FTP, filename: PurePosixPath, ctx: _AssemblyDownloadContext, last_activity: float, ) -> float: - """Download one assembly file, verify its MD5 checksum, and write an ``.md5`` sidecar if valid. + """Download one assembly file, verify its MD5 checksum in-stream, and write an ``.md5`` sidecar if valid. :param ftp: active FTP connection :type ftp: FTP - :param filename: remote file name within the current FTP working directory + :param filename: remote file name within the active FTP working directory :type filename: PurePosixPath :param ctx: local destination directory, MD5 table, and download stats :type ctx: _AssemblyDownloadContext - :param last_activity: monotonic timestamp of the previous FTP operation + :param last_activity: monotonic timestamp of the prior FTP operation :type last_activity: float :return: updated monotonic last-activity timestamp :rtype: float @@ -108,31 +142,34 @@ def _download_and_verify( local_file = ctx.dest_dir / filename expected_md5 = ctx.md5_checksums.get(filename) - def _validate_file(candidate_file: Path) -> bool: - if expected_md5: - if compute_file_checksum(candidate_file, "md5") == expected_md5: - (ctx.dest_dir / f"{filename}.md5").write_text(expected_md5, encoding="utf-8") - else: - logger.warning( - " MD5 mismatch for %s: expected %s", - filename, - expected_md5, - ) - return False - logger.debug(" MD5 verified: %s", filename) - else: - ctx.stats["files_without_checksum"] += 1 - return True - for attempt in range(1, MAX_DOWNLOAD_ATTEMPTS + 1): logger.debug(" Downloading %s (attempt %d/%d)", filename, attempt, MAX_DOWNLOAD_ATTEMPTS) - ftp_download_file(ftp, filename, local_file, retries=1) + actual_md5 = ftp_download_file( + ftp, + filename, + local_file, + retries=1, + checksum_fn="md5" if expected_md5 else None, + ) last_activity = time.monotonic() - if _validate_file(local_file): + if not expected_md5: + ctx.stats["files_without_checksum"] += 1 ctx.stats["files_downloaded"] += 1 return last_activity + if actual_md5 == expected_md5: + (ctx.dest_dir / f"{filename}.md5").write_text(expected_md5, encoding="utf-8") + logger.debug(" MD5 verified: %s", filename) + ctx.stats["files_downloaded"] += 1 + return last_activity + + logger.warning( + " MD5 mismatch for %s: expected %s", + filename, + expected_md5, + ) + ctx.stats["files_skipped_checksum_mismatch"] += 1 local_file.unlink(missing_ok=True) return last_activity @@ -155,7 +192,7 @@ def download_assembly_to_local( :type output_dir: Path :param ftp_host: FTP hostname :type ftp_host: str - :param ftp: optional existing FTP connection managed by the caller + :param ftp: optional caller-managed FTP connection :type ftp: FTP | None :return: JSON-serializable dictionary of download statistics :rtype: dict[str, Any] @@ -185,7 +222,7 @@ def download_assembly_to_local( (dest_dir / "md5checksums.txt").write_text(md5_text, encoding="utf-8") stats["files_downloaded"] += 1 - target_files = [f for f in files if any(f.match(f"**{s}") for s in FILE_FILTERS)] + target_files = [f for f in files if matches_assembly_file_filter(f)] last_activity = time.monotonic() ctx = _AssemblyDownloadContext(dest_dir=dest_dir, md5_checksums=md5_checksums, stats=stats) diff --git a/src/cdm_data_loaders/datasets/ncbi_ftp/constants.py b/src/cdm_data_loaders/datasets/ncbi_ftp/constants.py index 717e89c6..89aae5f5 100644 --- a/src/cdm_data_loaders/datasets/ncbi_ftp/constants.py +++ b/src/cdm_data_loaders/datasets/ncbi_ftp/constants.py @@ -1,16 +1,40 @@ -"""Constants and compiled regular expressions for NCBI FTP assembly paths and S3 prefixes.""" +"""Constants, source enums, and compiled regular expressions for NCBI FTP assembly paths and S3 prefixes.""" import re +from enum import StrEnum from pathlib import PurePosixPath +from types import MappingProxyType from typing import Final + +class Source(StrEnum): + """NCBI assembly database source.""" + + REFSEQ = "refseq" + GENBANK = "genbank" + + +DATABASE_ACC_PREFIX: Final[MappingProxyType[str, str]] = MappingProxyType( + { + Source.REFSEQ: "GCF_", + Source.GENBANK: "GCA_", + } +) + FTP_HOST: Final[str] = "ftp.ncbi.nlm.nih.gov" -# Extracts the database (GCF or GCA) and 3-digit parts from an accession ID -ACCESSION_PARTS_REGEX: Final[re.Pattern[str]] = re.compile(r"(GC[AF])_(\d{3})(\d{3})(\d{3})\.\d+.*") +# Matches a bare accession (GCF_000001215.4) or an assembly directory name (GCF_000001215.4_Release_6...) +ACCESSION_PARTS_REGEX: Final[re.Pattern[str]] = re.compile( + r"^(?P(?PGC[AF])_(?P\d{3})(?P\d{3})(?P\d{3})\.\d+)(?:_(?P[^/]+))?$" +) + +# Matches an assembly directory name with a required suffix after the versioned accession +ASSEMBLY_DIR_REGEX: Final[re.Pattern[str]] = re.compile(r"^(?PGC[AF]_\d{9}\.\d+)_(?P[^/]+)$") -# Extracts the full assembly directory, accession, and database (GCA or GCF) from an FTP path -ASSEMBLY_PATH_REGEX: Final[re.Pattern[str]] = re.compile(r"(((GC[AF])_\d{9}\.\d+)_[^/]+)/?.*$") +# Extracts the assembly directory, accession, and database (GCA or GCF) from an FTP or S3 path +ASSEMBLY_PATH_REGEX: Final[re.Pattern[str]] = re.compile( + r"(?:^|/)(?P(?P(?PGC[AF])_\d{9}\.\d+)_[^/]+)(?:/.*)?$" +) DEFAULT_STAGING_BUCKET: Final[PurePosixPath] = PurePosixPath("cts") DEFAULT_STAGING_KEY_PREFIX: Final[PurePosixPath] = PurePosixPath("staging") diff --git a/src/cdm_data_loaders/datasets/ncbi_ftp/download.py b/src/cdm_data_loaders/datasets/ncbi_ftp/download.py index 23541185..a53f6d61 100644 --- a/src/cdm_data_loaders/datasets/ncbi_ftp/download.py +++ b/src/cdm_data_loaders/datasets/ncbi_ftp/download.py @@ -7,7 +7,6 @@ from collections.abc import Callable from concurrent.futures import ThreadPoolExecutor, as_completed from datetime import UTC, datetime -from ftplib import error_temp from logging import Logger, getLogger from pathlib import Path, PurePosixPath from typing import Any, TypedDict, Unpack @@ -21,7 +20,8 @@ parse_assembly_path, ) from cdm_data_loaders.datasets.ncbi_ftp.constants import FTP_HOST -from cdm_data_loaders.io.ftp_client import ThreadLocalFTP +from cdm_data_loaders.datasets.ncbi_ftp.manifest import read_manifest_lines +from cdm_data_loaders.io.ftp_client import TRANSIENT_FTP_EXCEPTIONS, ThreadLocalFTP from cdm_data_loaders.io.s3.object_utils import get_object_bytes, upload_dir_files, upload_file logger: Logger = getLogger(__name__) @@ -102,8 +102,7 @@ def download_batch( on_assembly_downloaded = options.get("on_assembly_downloaded") progress_desc = options.get("progress_desc", "Downloading from NCBI FTP") - with manifest_path.open(encoding="utf-8") as fh: - assembly_paths = [PurePosixPath(line.strip()) for line in fh if line.strip() and not line.startswith("#")] + assembly_paths = [PurePosixPath(line) for line in read_manifest_lines(manifest_path)] if limit: assembly_paths = assembly_paths[:limit] @@ -120,7 +119,7 @@ def _download_one(path: PurePosixPath) -> tuple[str, Exception | None]: nonlocal success_count @retry( - retry=retry_if_exception_type((error_temp, BrokenPipeError, ConnectionResetError, EOFError)), + retry=retry_if_exception_type(TRANSIENT_FTP_EXCEPTIONS), stop=stop_after_attempt(5), wait=wait_exponential(multiplier=1, min=5, max=60), reraise=True, diff --git a/src/cdm_data_loaders/datasets/ncbi_ftp/manifest.py b/src/cdm_data_loaders/datasets/ncbi_ftp/manifest.py index bd10f046..18b663ac 100644 --- a/src/cdm_data_loaders/datasets/ncbi_ftp/manifest.py +++ b/src/cdm_data_loaders/datasets/ncbi_ftp/manifest.py @@ -1,9 +1,9 @@ -"""S3 store scanning, FTP checksum verification, and manifest file writers for NCBI FTP.""" +"""S3 store scanning, FTP checksum verification, and manifest readers and writers for NCBI FTP.""" import contextlib import json import time -from collections.abc import Callable, Sequence +from collections.abc import Callable, Iterable, Sequence from datetime import UTC, datetime from ftplib import FTP from logging import Logger, getLogger @@ -12,12 +12,12 @@ from cdm_data_loaders.datasets.ncbi_ftp.assembly import ( build_accession_path, + extract_accession, filter_assembly_checksums, parse_assembly_path, ) -from cdm_data_loaders.datasets.ncbi_ftp.constants import FTP_HOST +from cdm_data_loaders.datasets.ncbi_ftp.constants import DATABASE_ACC_PREFIX, FTP_HOST, Source from cdm_data_loaders.datasets.ncbi_ftp.summary import ( - DATABASE_ACC_PREFIX, AssemblyRecord, DiffResult, ftp_dir_from_url, @@ -36,17 +36,57 @@ class VerifyCandidatesOptions(TypedDict, total=False): progress_callback: Callable[[int, int, str], None] | None -def _extract_accession_dir_and_id_from_s3_key(key: PurePosixPath) -> tuple[str, str]: +def read_manifest_lines(source: Path | Iterable[str]) -> list[str]: + """Read non-empty, non-comment manifest entries from a file path or line iterable. + + :param source: manifest file path or iterable of raw lines + :type source: Path | Iterable[str] + :return: stripped manifest lines excluding blank lines and ``#`` comments + :rtype: list[str] + """ + if isinstance(source, Path): + with source.open(encoding="utf-8") as fh: + return [stripped for line in fh if (stripped := line.strip()) and not stripped.startswith("#")] + return [stripped for line in source if (stripped := line.strip()) and not stripped.startswith("#")] + + +def filter_unpromoted_manifest_lines( + text: str, + promoted_accessions: set[str], +) -> tuple[list[str], int]: + """Filter raw manifest text, removing entries whose exact accession is in ``promoted_accessions``. + + :param text: raw UTF-8 manifest text + :type text: str + :param promoted_accessions: set of promoted accession IDs (for example, ``{"GCF_000001215.4"}``) + :type promoted_accessions: set[str] + :return: tuple of ``(remaining_entries, total_entries)`` after skipping blank and comment lines + :rtype: tuple[list[str], int] + """ + entries = read_manifest_lines(text.splitlines()) + remaining: list[str] = [] + for entry in entries: + try: + entry_acc = extract_accession(entry) + except ValueError: + remaining.append(entry) + continue + if entry_acc not in promoted_accessions: + remaining.append(entry) + return remaining, len(entries) + + +def _extract_accession_dir_and_id_from_s3_key(key: str | PurePosixPath) -> tuple[str, str]: """Extract ``(assembly_dir, accession)`` from an S3 object key. :param key: S3 object key containing an assembly directory segment - :type key: PurePosixPath + :type key: str | PurePosixPath :return: tuple of ``(assembly_dir_name, accession)`` :rtype: tuple[str, str] :raises ValueError: if ``key`` does not contain a valid assembly directory """ try: - _db, assembly_dir, accession = parse_assembly_path(key) + _db, assembly_dir, accession = parse_assembly_path(PurePosixPath(key)) except ValueError as exc: msg = f"Could not parse S3 key for accession info: {key}" raise ValueError(msg) from exc @@ -57,10 +97,10 @@ def scan_store_to_synthetic_summary( bucket: PurePosixPath, key_prefix: PurePosixPath, release_date: str, - database: str = "refseq", + database: str = Source.REFSEQ, progress_callback: Callable[[int, str], None] | None = None, ) -> dict[str, AssemblyRecord]: - """Scan an S3 store and build a synthetic assembly summary from existing objects. + """Scan an S3 store and build a synthetic assembly summary from present objects. :param bucket: S3 bucket name :type bucket: PurePosixPath @@ -129,7 +169,7 @@ def _fetch_accession_checksums_from_ftp( ) -> tuple[dict[PurePosixPath, str], float, FTP]: """Fetch and filter ``md5checksums.txt`` for an assembly URL from NCBI FTP. - :param ftp: active FTP client or ``None`` to connect lazily + :param ftp: active FTP client or ``None`` to connect on demand :type ftp: FTP | None :param ftp_host: NCBI FTP hostname :type ftp_host: str @@ -196,7 +236,7 @@ def verify_transfer_candidates( :param accessions: candidate accessions (new + updated from diff) :type accessions: list[str] - :param current_assemblies: parsed current assembly summary + :param current_assemblies: parsed assembly summary :type current_assemblies: dict[str, AssemblyRecord] :param bucket: S3 bucket name :type bucket: PurePosixPath @@ -281,11 +321,11 @@ def write_transfer_manifest( :param diff: computed diff result :type diff: DiffResult - :param current_assemblies: parsed current assembly summary + :param current_assemblies: parsed assembly summary :type current_assemblies: dict[str, AssemblyRecord] :param output_path: destination manifest file path :type output_path: Path - :param ftp_host: FTP hostname for URL validation + :param ftp_host: FTP hostname for URL checks :type ftp_host: str :return: list of FTP paths written :rtype: list[PurePosixPath] diff --git a/src/cdm_data_loaders/datasets/ncbi_ftp/promote.py b/src/cdm_data_loaders/datasets/ncbi_ftp/promote.py index f5ea23aa..4fc52ca2 100644 --- a/src/cdm_data_loaders/datasets/ncbi_ftp/promote.py +++ b/src/cdm_data_loaders/datasets/ncbi_ftp/promote.py @@ -6,15 +6,21 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from dataclasses import dataclass from datetime import UTC, datetime -from http import HTTPStatus from logging import Logger, getLogger from pathlib import Path, PurePosixPath from typing import Any, Final, TypedDict, Unpack -import botocore.exceptions import tqdm -from cdm_data_loaders.datasets.ncbi_ftp.assembly import build_accession_path, parse_assembly_path +from cdm_data_loaders.datasets.ncbi_ftp.assembly import ( + build_accession_path, + extract_accession, + parse_assembly_path, +) +from cdm_data_loaders.datasets.ncbi_ftp.manifest import ( + filter_unpromoted_manifest_lines, + read_manifest_lines, +) from cdm_data_loaders.datasets.ncbi_ftp.metadata import ( DescriptorResource, archive_descriptor, @@ -23,11 +29,11 @@ upload_descriptor, ) from cdm_data_loaders.io.s3.object_utils import ( - NOT_FOUND_ERROR_CODES, copy_object, delete_objects, download_file, get_object_bytes, + get_object_bytes_if_exists, list_objects, object_exists, put_object_bytes, @@ -36,6 +42,7 @@ logger: Logger = getLogger(__name__) +ASSEMBLY_DIR_SEGMENT_INDEX: Final[int] = 5 MAX_DRY_RUN_LOGS: Final[int] = 10 MAX_TRANSFER_WORKERS: Final[int] = 32 @@ -369,22 +376,27 @@ def _get_source_dest_pairs_for_accession( release_tag: str, archive_reason: str, ) -> list[tuple[PurePosixPath, PurePosixPath]]: - """Build ``(source_key, archive_key)`` pairs for all S3 objects matching ``accession``.""" + """Build ``(source_key, archive_key)`` pairs for all S3 objects belonging to ``accession``.""" source_prefix = _get_accession_path_prefix(accession, lakehouse_key_prefix) if not source_prefix: return [] + target_accession = extract_accession(accession) matching_objs: list[dict[str, Any]] = list_objects(f"{lakehouse_bucket / source_prefix}") - return [ - ( - PurePosixPath(obj["Key"]), - lakehouse_key_prefix - / "archive" - / release_tag - / archive_reason - / PurePosixPath(obj["Key"]).relative_to(lakehouse_key_prefix), + pairs: list[tuple[PurePosixPath, PurePosixPath]] = [] + for obj in matching_objs: + src_key = PurePosixPath(obj["Key"]) + rel_key = src_key.relative_to(lakehouse_key_prefix) + if len(rel_key.parts) > ASSEMBLY_DIR_SEGMENT_INDEX: + with contextlib.suppress(ValueError): + if extract_accession(rel_key.parts[ASSEMBLY_DIR_SEGMENT_INDEX]) != target_accession: + continue + pairs.append( + ( + src_key, + lakehouse_key_prefix / "archive" / release_tag / archive_reason / rel_key, + ) ) - for obj in matching_objs - ] + return pairs def _archive_objects( @@ -438,8 +450,7 @@ def _archive_assemblies( dry_run = options.get("dry_run", False) archived = 0 - with Path(manifest_local_path).open(encoding="utf-8") as fh: - accessions = [line.strip() for line in fh if line.strip()] + accessions = read_manifest_lines(Path(manifest_local_path)) dry_run_log_count = 0 for accession in tqdm.tqdm(accessions, unit="accession", desc="Archiving"): @@ -492,22 +503,17 @@ def _trim_manifest( ) -> None: """Remove promoted accessions from ``manifest_s3_key`` in ``staging_bucket``.""" s3_path = str(staging_bucket / manifest_s3_key) - try: - raw_bytes = get_object_bytes(s3_path) - except botocore.exceptions.ClientError as exc: - error_code = exc.response.get("Error", {}).get("Code") - status_code = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") - if error_code in NOT_FOUND_ERROR_CODES or status_code == HTTPStatus.NOT_FOUND: - logger.warning("Manifest not found in S3 (s3://%s/%s) — skipping trim", staging_bucket, manifest_s3_key) - return - raise - - lines = raw_bytes.decode("utf-8").splitlines(keepends=True) - remaining = [line for line in lines if line.strip() and not any(acc in line for acc in promoted_accessions)] - put_object_bytes("".join(remaining).encode("utf-8"), s3_path, content_type="text/plain") + raw_bytes = get_object_bytes_if_exists(s3_path) + if raw_bytes is None: + logger.warning("Manifest not found in S3 (s3://%s/%s) — skipping trim", staging_bucket, manifest_s3_key) + return + + remaining, total_entries = filter_unpromoted_manifest_lines(raw_bytes.decode("utf-8"), promoted_accessions) + payload = "".join(f"{entry}\n" for entry in remaining).encode("utf-8") + put_object_bytes(payload, s3_path, content_type="text/plain") logger.info( "Trimmed manifest: %d -> %d entries (%d promoted)", - len(lines), + total_entries, len(remaining), - len(lines) - len(remaining), + total_entries - len(remaining), ) diff --git a/src/cdm_data_loaders/datasets/ncbi_ftp/summary.py b/src/cdm_data_loaders/datasets/ncbi_ftp/summary.py index 6662cdd2..a253957d 100644 --- a/src/cdm_data_loaders/datasets/ncbi_ftp/summary.py +++ b/src/cdm_data_loaders/datasets/ncbi_ftp/summary.py @@ -5,23 +5,21 @@ from dataclasses import dataclass, field from logging import Logger, getLogger from pathlib import Path, PurePosixPath +from types import MappingProxyType from typing import Final from urllib.parse import urlsplit -from cdm_data_loaders.datasets.ncbi_ftp.constants import ACCESSION_PARTS_REGEX, FTP_HOST +from cdm_data_loaders.datasets.ncbi_ftp.constants import ACCESSION_PARTS_REGEX, FTP_HOST, Source from cdm_data_loaders.io.ftp_client import ftp_retrieve_text, ftp_session logger: Logger = getLogger(__name__) -DATABASE_ACC_PREFIX: Final[dict[str, str]] = { - "refseq": "GCF_", - "genbank": "GCA_", -} - -SUMMARY_FTP_PATHS: Final[dict[str, PurePosixPath]] = { - "refseq": PurePosixPath("/") / "genomes" / "ASSEMBLY_REPORTS" / "assembly_summary_refseq.txt", - "genbank": PurePosixPath("/") / "genomes" / "ASSEMBLY_REPORTS" / "assembly_summary_genbank.txt", -} +SUMMARY_FTP_PATHS: Final[MappingProxyType[str, PurePosixPath]] = MappingProxyType( + { + Source.REFSEQ: PurePosixPath("/") / "genomes" / "ASSEMBLY_REPORTS" / "assembly_summary_refseq.txt", + Source.GENBANK: PurePosixPath("/") / "genomes" / "ASSEMBLY_REPORTS" / "assembly_summary_genbank.txt", + } +) ACCESSION_COL: Final[int] = 0 STATUS_COL: Final[int] = 10 @@ -43,7 +41,7 @@ class AssemblyRecord: @dataclass class DiffResult: - """Result of comparing current and previous assembly summaries.""" + """Result of comparing two NCBI assembly summary states.""" new: list[str] = field(default_factory=list) updated: list[str] = field(default_factory=list) @@ -54,7 +52,7 @@ class DiffResult: def ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> PurePosixPath: """Convert an FTP or HTTPS URL from the assembly summary to an FTP directory path. - :param ftp_url: full FTP or HTTPS URL from an assembly summary row + :param ftp_url: FTP or HTTPS URL from an assembly summary row :type ftp_url: str :param ftp_host: expected FTP hostname :type ftp_host: str @@ -73,7 +71,7 @@ def ftp_dir_from_url(ftp_url: str, ftp_host: str = FTP_HOST) -> PurePosixPath: raise ValueError(msg) -def download_assembly_summary(database: str = "refseq", ftp_host: str = FTP_HOST) -> str: +def download_assembly_summary(database: str = Source.REFSEQ, ftp_host: str = FTP_HOST) -> str: """Download the assembly summary TSV from NCBI FTP. :param database: ``"refseq"`` or ``"genbank"`` @@ -154,7 +152,7 @@ def get_latest_assembly_paths( :param assemblies: parsed assembly records :type assemblies: dict[str, AssemblyRecord] - :param ftp_host: FTP hostname for URL validation + :param ftp_host: FTP hostname for URL checks :type ftp_host: str :return: list of ``(accession, ftp_dir_path)`` pairs :rtype: list[tuple[str, PurePosixPath]] @@ -177,7 +175,7 @@ def accession_prefix(accession: str) -> str: :raises ValueError: if ``accession`` cannot be parsed """ if m := ACCESSION_PARTS_REGEX.match(accession): - return m.group(2) + return m.group("p1") msg = f"Could not parse accession: {accession}" raise ValueError(msg) @@ -214,11 +212,11 @@ def compute_diff( previous_assemblies: dict[str, AssemblyRecord] | None = None, previous_accessions: set[str] | None = None, ) -> DiffResult: - """Compute new, updated, replaced, and suppressed assemblies between current and previous state. + """Compute new, updated, replaced, and suppressed assemblies between two states. - :param current: parsed current NCBI assembly summary + :param current: parsed NCBI assembly summary :type current: dict[str, AssemblyRecord] - :param previous_assemblies: parsed previous summary, or ``None`` when using store-scan fallback + :param previous_assemblies: parsed baseline summary, or ``None`` when using store-scan fallback :type previous_assemblies: dict[str, AssemblyRecord] | None :param previous_accessions: set of known accessions from store scan when ``previous_assemblies`` is ``None`` :type previous_accessions: set[str] | None diff --git a/src/cdm_data_loaders/datasets/ncbi_ftp/updates.py b/src/cdm_data_loaders/datasets/ncbi_ftp/updates.py index 16461c4e..3cb42c64 100644 --- a/src/cdm_data_loaders/datasets/ncbi_ftp/updates.py +++ b/src/cdm_data_loaders/datasets/ncbi_ftp/updates.py @@ -1,9 +1,8 @@ """Filter NCBI FTP assembly modification times by date, RefSeq/GenBank source, and assemblies of interest.""" +import itertools import math import operator -import re -from collections.abc import Iterable from datetime import date from enum import StrEnum from logging import Logger, getLogger @@ -12,19 +11,13 @@ import pandas as pd +from cdm_data_loaders.datasets.ncbi_ftp.constants import ASSEMBLY_DIR_REGEX, DATABASE_ACC_PREFIX, Source + logger: Logger = getLogger(__name__) -ASSEMBLY_DIR_RE: Final[re.Pattern[str]] = re.compile(r"^(GC[AF]_\d{9}\.\d+)_(.+)$") BATCH_SIZE: Final[int] = 1000 -class Source(StrEnum): - """NCBI assembly database source.""" - - REFSEQ = "refseq" - GENBANK = "genbank" - - class Period(StrEnum): """Temporal partition relative to the modification date threshold.""" @@ -42,13 +35,13 @@ def parse_assembly_dir(assembly_dir: str) -> tuple[str, str]: :type assembly_dir: str :return: tuple of ``(accession_id, assembly_name)`` :rtype: tuple[str, str] - :raises ValueError: if ``assembly_dir`` does not match the expected NCBI directory pattern + :raises ValueError: if ``assembly_dir`` does not match the NCBI directory pattern """ - match = ASSEMBLY_DIR_RE.match(assembly_dir) + match = ASSEMBLY_DIR_REGEX.match(assembly_dir) if not match: msg = f"assembly_dir '{assembly_dir}' does not match expected pattern" raise ValueError(msg) - return match.group(1), match.group(2) + return match.group("accession"), match.group("name") def infer_source(assembly_id: str) -> Source: @@ -60,10 +53,9 @@ def infer_source(assembly_id: str) -> Source: :rtype: Source :raises ValueError: if ``assembly_id`` lacks a ``GCF_`` or ``GCA_`` prefix """ - if assembly_id.startswith("GCF_"): - return Source.REFSEQ - if assembly_id.startswith("GCA_"): - return Source.GENBANK + for source_name, prefix in DATABASE_ACC_PREFIX.items(): + if assembly_id.startswith(prefix): + return Source(source_name) msg = f"Unrecognized assembly_id prefix: {assembly_id}" raise ValueError(msg) @@ -212,20 +204,6 @@ def save_tsv(df: pd.DataFrame, path: Path) -> None: logger.info("Wrote %d rows -> %s", len(df), path) -def batch(items: list[str], size: int) -> Iterable[list[str]]: - """Yield contiguous chunks of ``items`` of length up to ``size``. - - :param items: list of strings to batch - :type items: list[str] - :param size: maximum batch size - :type size: int - :return: iterable of string batches - :rtype: Iterable[list[str]] - """ - for i in range(0, len(items), size): - yield items[i : i + size] - - def batch_count(n_items: int, batch_size: int) -> int: """Compute the number of batches needed for ``n_items`` at ``batch_size``. @@ -254,7 +232,7 @@ def compute_batch_width(splits: Splits, batch_size: int) -> int: def write_id_batches(ids: list[str], out_dir: Path, prefix: str, batch_size: int, width: int) -> int: - """Write sequentially numbered batch text files of assembly IDs under ``out_dir``. + """Write numbered batch text files of assembly IDs under ``out_dir``. :param ids: ordered assembly IDs :type ids: list[str] @@ -270,7 +248,7 @@ def write_id_batches(ids: list[str], out_dir: Path, prefix: str, batch_size: int :rtype: int """ n_written = 0 - for i, chunk in enumerate(batch(ids, batch_size), start=1): + for i, chunk in enumerate(itertools.batched(ids, batch_size, strict=False), start=1): file_num = str(i).zfill(width) out_path = out_dir / f"{prefix}-{file_num}.txt" out_path.write_text("\n".join(chunk) + "\n", encoding="utf-8") diff --git a/src/cdm_data_loaders/datasets/uniprot/common.py b/src/cdm_data_loaders/datasets/uniprot/common.py index 3bb1f091..14d4cf20 100644 --- a/src/cdm_data_loaders/datasets/uniprot/common.py +++ b/src/cdm_data_loaders/datasets/uniprot/common.py @@ -1,9 +1,39 @@ -"""Shared mapping, date, and identifier helpers for UniProt dataset transforms.""" +"""Shared mapping, date, CURIE, and CDM entry-transform helpers for UniProt dataset transforms.""" import datetime as dt -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence +from logging import Logger +from types import MappingProxyType from typing import Any, Final +from cdm_data_loaders.core.buffer import make_rejected_table_result +from cdm_data_loaders.core.registry import RecordTransformFn, TransformContext + +DEFAULT_ENTRY_TABLE_NAME: Final[str] = "entry" +UNIPROT_CURIE_NAMESPACE: Final[str] = "uniprot" +UNIPROT_CURIE_PREFIX: Final[str] = f"{UNIPROT_CURIE_NAMESPACE}:" +UNIREF_CURIE_NAMESPACE: Final[str] = "uniref" +UNIREF_CURIE_PREFIX: Final[str] = f"{UNIREF_CURIE_NAMESPACE}:" +NCBI_TAXON_PREFIX: Final[str] = "NCBITaxon" + +PREFIX_TRANSLATION: Final[MappingProxyType[str, str]] = MappingProxyType( + { + "UniProtKB": UNIPROT_CURIE_NAMESPACE, + "UniProtKB/Swiss-Prot": UNIPROT_CURIE_NAMESPACE, + "UniProtKB/TrEMBL": UNIPROT_CURIE_NAMESPACE, + "UniProtKB accession": UNIPROT_CURIE_NAMESPACE, + "UniProtKB ID": "uniprot_name", + "UniParc ID": "uniparc", + "UniRef90 ID": UNIREF_CURIE_NAMESPACE, + "UniRef50 ID": UNIREF_CURIE_NAMESPACE, + "UniRef100 ID": UNIREF_CURIE_NAMESPACE, + "NCBI Taxonomy": NCBI_TAXON_PREFIX, + "NCBI taxonomy": NCBI_TAXON_PREFIX, + "GeneID": "NCBIGene", + "pubmed": "PMID", + } +) + ENSEMBL_PREFIX_TYPES: Final[dict[str, str]] = { "ENST": "transcript", "ENSP": "protein sequence", @@ -54,4 +84,111 @@ def ensembl_type(xref: str) -> str: :return: entity category label :rtype: str """ - return ENSEMBL_PREFIX_TYPES.get(xref[0:4], ENSEMBL_PREFIX_TYPES.get(xref[0:5], "sequence")) + return ENSEMBL_PREFIX_TYPES.get(xref[0:5], ENSEMBL_PREFIX_TYPES.get(xref[0:4], "sequence")) + + +def format_curie(db: str, accession: str, *, fallback_upper: bool = False) -> str: + """Format ``db`` and ``accession`` into a BioRegistry CURIE string. + + :param db: source database name or reference type + :type db: str + :param accession: identifier within ``db`` + :type accession: str + :param fallback_upper: when ``True``, uppercase ``db`` if it is absent from ``PREFIX_TRANSLATION`` + :type fallback_upper: bool + :return: CURIE string ``:`` + :rtype: str + """ + fallback = db.upper() if fallback_upper else db + return f"{PREFIX_TRANSLATION.get(db, fallback)}:{accession}" + + +def resolve_timestamp(timestamp: dt.datetime | None) -> dt.datetime: + """Return ``timestamp`` when provided, else ``datetime.now(UTC)``. + + :param timestamp: explicit UTC timestamp or ``None`` + :type timestamp: dt.datetime | None + :return: resolved UTC timestamp + :rtype: dt.datetime + """ + return timestamp if timestamp is not None else dt.datetime.now(dt.UTC) + + +def reject_cdm_entry( + entry: Mapping[str, Any], + error_detail: str, + ctx: TransformContext | None = None, + *, + default_table: str = DEFAULT_ENTRY_TABLE_NAME, +) -> dict[str, list[dict[str, Any]]]: + """Build a ``{_rejected: [row]}`` mapping preserving ``ctx.table_name`` and ``ctx.source_file``. + + :param entry: input XSD-converted entry dictionary + :type entry: Mapping[str, Any] + :param error_detail: human-readable rejection message + :type error_detail: str + :param ctx: optional transform context carrying table name and source file + :type ctx: TransformContext | None + :param default_table: fallback table name when ``ctx`` is ``None`` + :type default_table: str + :return: single-row ``_rejected`` table result mapping + :rtype: dict[str, list[dict[str, Any]]] + """ + table_name = ctx.table_name if ctx is not None else default_table + source_file = ctx.source_file if ctx is not None else None + return make_rejected_table_result( + table_name, + entry, + error_detail, + source_file=source_file, + ) + + +def run_cdm_entry_parser( + entry: Mapping[str, Any], + parse_fn: Callable[[], dict[str, list[dict[str, Any]]]], + *, + ctx: TransformContext | None = None, + logger: Logger, + error_message: str, +) -> dict[str, list[dict[str, Any]]]: + """Execute ``parse_fn`` and route unexpected exceptions to ``_rejected`` with ``ctx`` provenance. + + :param entry: input XSD-converted entry dictionary + :type entry: Mapping[str, Any] + :param parse_fn: zero-argument callable producing routed CDM table rows + :type parse_fn: Callable[[], dict[str, list[dict[str, Any]]]] + :param ctx: optional transform context carrying table name and source file + :type ctx: TransformContext | None + :param logger: module logger used to log unexpected exceptions + :type logger: Logger + :param error_message: log message emitted on unexpected parse errors + :type error_message: str + :return: routed CDM table rows or a single-row ``_rejected`` mapping + :rtype: dict[str, list[dict[str, Any]]] + """ + try: + return parse_fn() + except Exception as exc: + logger.exception(error_message) + return reject_cdm_entry(entry, str(exc), ctx) + + +def make_cdm_entry_transform( + entry_fn: Callable[[Mapping[str, Any], TransformContext | None], dict[str, list[dict[str, Any]]]], +) -> RecordTransformFn: + """Wrap ``entry_fn(entry, ctx)`` into a context-aware ``RecordTransformFn``. + + :param entry_fn: callable accepting ``(entry, ctx)`` and returning routed CDM table rows + :type entry_fn: Callable[[Mapping[str, Any], TransformContext | None], dict[str, list[dict[str, Any]]]] + :return: ``RecordTransformFn`` accepting ``(entry, ctx=None)`` + :rtype: RecordTransformFn + """ + + def _transform( + entry: dict[str, Any], + ctx: TransformContext | None = None, + ) -> dict[str, list[dict[str, Any]]]: + return entry_fn(entry, ctx) + + return _transform diff --git a/src/cdm_data_loaders/datasets/uniprot/idmapping.py b/src/cdm_data_loaders/datasets/uniprot/idmapping.py index a673ab89..e0fd026f 100644 --- a/src/cdm_data_loaders/datasets/uniprot/idmapping.py +++ b/src/cdm_data_loaders/datasets/uniprot/idmapping.py @@ -8,6 +8,7 @@ from cdm_data_loaders.core.registry import ArrowSchemaRegistry, TableTransformFn, TransformContext from cdm_data_loaders.core.transforms.routing import emit_raw_table +from cdm_data_loaders.datasets.uniprot.common import UNIPROT_CURIE_PREFIX from cdm_data_loaders.schemas.uniprot.idmapping.schema import ( DB_REFERENCE_ID_COL, DB_REFERENCE_TYPE_COL, @@ -18,8 +19,6 @@ UNIPROT_ID_COL, ) -UNIPROT_CURIE_PREFIX: Final[str] = "uniprot:" - UNIPROT_IDMAPPING_OUTPUT_SCHEMAS: Final[ArrowSchemaRegistry] = ArrowSchemaRegistry( registry={ RAW_ID_MAPPING_TABLE_NAME: RAW_ID_MAPPING_SCHEMA, diff --git a/src/cdm_data_loaders/datasets/uniprot/kb.py b/src/cdm_data_loaders/datasets/uniprot/kb.py index 14a2d284..fefd503c 100644 --- a/src/cdm_data_loaders/datasets/uniprot/kb.py +++ b/src/cdm_data_loaders/datasets/uniprot/kb.py @@ -1,22 +1,31 @@ """UniProtKB XML entry transforms and CDM PyArrow output schemas.""" import datetime as dt -import functools from collections.abc import Mapping from logging import Logger, getLogger from typing import Any, Final import pyarrow as pa -from cdm_data_loaders.core.buffer import make_rejected_table_result -from cdm_data_loaders.core.registry import ArrowSchemaRegistry, RecordTransformFn -from cdm_data_loaders.datasets.uniprot.common import ensembl_type, format_date, iter_mappings +from cdm_data_loaders.core.registry import ArrowSchemaRegistry, RecordTransformFn, TransformContext +from cdm_data_loaders.datasets.uniprot.common import ( + NCBI_TAXON_PREFIX, + UNIPROT_CURIE_PREFIX, + ensembl_type, + format_curie, + format_date, + iter_mappings, + make_cdm_entry_transform, + reject_cdm_entry, + resolve_timestamp, + run_cdm_entry_parser, +) DB: Final[str] = "db" DESCRIPTION: Final[str] = "description" ENTITY_ID: Final[str] = "entity_id" NAME: Final[str] = "name" -NCBI_TAXON: Final[str] = "NCBITaxon" +NCBI_TAXON: Final[str] = NCBI_TAXON_PREFIX XREF: Final[str] = "xref" # XML namespace mapping for UniProt entries @@ -24,16 +33,6 @@ NS: Final[dict[str, str]] = {"ns": HTTPS_NS, "": HTTPS_NS} ENTRY_XML_TAG: Final[str] = f"{{{HTTPS_NS}}}entry" -# CURIE prefixes -PREFIX_TRANSLATION: Final[dict[str, str]] = { - "UniProtKB": "UniProt", - "UniProtKB/Swiss-Prot": "UniProt", - "UniProtKB/TrEMBL": "UniProt", - "NCBI Taxonomy": NCBI_TAXON, - "GeneID": "NCBIGene", - "pubmed": "PMID", -} - REF_DB_PRIORITY_ORDER: Final[tuple[str, ...]] = ("doi", "pmcid", "pmid", "pubmed", "agricola") ENTITY_SCHEMA: Final[pa.Schema] = pa.schema( @@ -413,7 +412,7 @@ def parse_references(entry: Mapping[str, Any]) -> list[dict[str, str]]: for ref_type in REF_DB_PRIORITY_ORDER: refs = [ref[XREF] for ref in citation_refs if ref[DB] == ref_type] if refs: - publications.add(f"{PREFIX_TRANSLATION.get(ref_type, ref_type.upper())}:{refs[0]}") + publications.add(format_curie(ref_type, refs[0], fallback_upper=True)) ref_found = True break @@ -426,6 +425,8 @@ def parse_references(entry: Mapping[str, Any]) -> list[dict[str, str]]: def parse_uniprot_entry( entry: Mapping[str, Any], timestamp: dt.datetime, + *, + ctx: TransformContext | None = None, ) -> dict[str, list[dict[str, Any]]]: """Parse an XSD-converted UniProt entry dictionary into CDM schema tables. @@ -433,17 +434,20 @@ def parse_uniprot_entry( :type entry: Mapping[str, Any] :param timestamp: UTC timestamp assigned to ``entity.updated`` :type timestamp: dt.datetime + :param ctx: optional transform context carrying table name and source file + :type ctx: TransformContext | None :return: dictionary of parsed table rows keyed by destination table name :rtype: dict[str, list[dict[str, Any]]] """ - try: + + def _parse() -> dict[str, list[dict[str, Any]]]: accessions = _extract_accessions(entry) if not accessions: logger.error("No accession found for entry") - return make_rejected_table_result("entry", entry, "No accession found") + return reject_cdm_entry(entry, "No accession found", ctx) first_acc = accessions[0] - uniprot_id = f"uniprot:{first_acc}" + uniprot_id = f"{UNIPROT_CURIE_PREFIX}{first_acc}" raw_version = entry.get("_version") entity = { ENTITY_ID: uniprot_id, @@ -468,9 +472,14 @@ def parse_uniprot_entry( "protein": [{"protein_id": uniprot_id, **parse_protein_info(entry)}], "entity_x_publication": [{ENTITY_ID: uniprot_id, **e} for e in parse_references(entry)], } - except Exception as e: - logger.exception("Parse error while processing entry") - return make_rejected_table_result("entry", entry, str(e)) + + return run_cdm_entry_parser( + entry, + _parse, + ctx=ctx, + logger=logger, + error_message="Parse error while processing entry", + ) def make_uniprot_cdm_transform( @@ -483,5 +492,12 @@ def make_uniprot_cdm_transform( :return: ``RecordTransformFn`` transforming ``entry`` records into CDM table rows :rtype: RecordTransformFn """ - resolved_ts = timestamp if timestamp is not None else dt.datetime.now(dt.UTC) - return functools.partial(parse_uniprot_entry, timestamp=resolved_ts) + resolved_ts = resolve_timestamp(timestamp) + + def _parse_entry( + entry: Mapping[str, Any], + ctx: TransformContext | None, + ) -> dict[str, list[dict[str, Any]]]: + return parse_uniprot_entry(entry, resolved_ts, ctx=ctx) + + return make_cdm_entry_transform(_parse_entry) diff --git a/src/cdm_data_loaders/datasets/uniprot/uniref.py b/src/cdm_data_loaders/datasets/uniprot/uniref.py index bbb699ba..320a014b 100644 --- a/src/cdm_data_loaders/datasets/uniprot/uniref.py +++ b/src/cdm_data_loaders/datasets/uniprot/uniref.py @@ -1,32 +1,30 @@ """UniRef XML entry transforms and CDM PyArrow output schemas.""" import datetime as dt -import functools from collections.abc import Mapping from logging import Logger, getLogger from typing import Any, Final import pyarrow as pa -from cdm_data_loaders.core.buffer import make_rejected_table_result -from cdm_data_loaders.core.registry import ArrowSchemaRegistry, RecordTransformFn -from cdm_data_loaders.datasets.uniprot.common import format_date, iter_mappings +from cdm_data_loaders.core.registry import ArrowSchemaRegistry, RecordTransformFn, TransformContext +from cdm_data_loaders.datasets.uniprot.common import ( + UNIPROT_CURIE_PREFIX, + UNIREF_CURIE_PREFIX, + format_curie, + format_date, + iter_mappings, + make_cdm_entry_transform, + reject_cdm_entry, + resolve_timestamp, + run_cdm_entry_parser, +) UNIREF_URL: Final[str] = "http://uniprot.org/uniref" ENTRY_XML_TAG: Final[str] = f"{{{UNIREF_URL}}}entry" UNIREF: Final[str] = "UniRef" ENTITY_ID: Final[str] = "entity_id" -PREFIX_TRANSLATION: Final[dict[str, str]] = { - "UniProtKB ID": "uniprot_name", - "UniProtKB accession": "uniprot", - "UniParc ID": "uniparc", - "UniRef90 ID": "uniref", - "UniRef50 ID": "uniref", - "UniRef100 ID": "uniref", - "NCBI taxonomy": "NCBITaxon", -} - CLUSTER_SCHEMA: Final[pa.Schema] = pa.schema( [ pa.field("cluster_id", pa.string()), @@ -78,7 +76,7 @@ def generate_dbxref(db: str, acc: str) -> str: :return: CURIE-formatted identifier :rtype: str """ - return f"{PREFIX_TRANSLATION.get(db, db)}:{acc}" + return format_curie(db, acc) def extract_cluster(entry: Mapping[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]: @@ -92,7 +90,7 @@ def extract_cluster(entry: Mapping[str, Any]) -> tuple[dict[str, Any], dict[str, entry_id = entry.get("_id") if not entry_id: return ({}, {}) - cluster_id = f"uniref:{entry_id}" + cluster_id = f"{UNIREF_CURIE_PREFIX}{entry_id}" updated_str = format_date(entry.get("_updated")) cluster: dict[str, Any] = { "cluster_id": cluster_id, @@ -143,7 +141,7 @@ def extract_cross_refs( None, ) if uniprot_id: - entity_id = f"uniprot:{uniprot_id}" + entity_id = f"{UNIPROT_CURIE_PREFIX}{uniprot_id}" else: logger.warning("Could not find UniProt accession for UniProtKB ID %s", entity_xref) @@ -165,6 +163,8 @@ def parse_uniref_entry( entry: Mapping[str, Any], timestamp: dt.datetime, uniref_variant: str, + *, + ctx: TransformContext | None = None, ) -> dict[str, list[dict[str, Any]]]: """Parse a single XSD-converted UniRef ``entry`` dictionary into CDM table rows. @@ -174,36 +174,48 @@ def parse_uniref_entry( :type timestamp: dt.datetime :param uniref_variant: protocol label for the UniRef variant (for example, ``"UniRef 50"``) :type uniref_variant: str + :param ctx: optional transform context carrying table name and source file + :type ctx: TransformContext | None :return: dictionary of parsed table rows keyed by destination table name :rtype: dict[str, list[dict[str, Any]]] """ - cluster, entity = extract_cluster(entry) - cluster_id = cluster.get("cluster_id") - if not cluster_id: - logger.debug("No cluster ID found in entry") - return make_rejected_table_result("entry", entry, "No cluster ID found in entry") - - entity["updated"] = timestamp - cluster["protocol"] = uniref_variant - - cluster_members = [ - *( - extract_cross_refs(repr_db, cluster_id, is_representative=True) - for repr_member in iter_mappings(entry.get("representativeMember")) - for repr_db in iter_mappings(repr_member.get("dbReference")) - ), - *( - extract_cross_refs(mem_db, cluster_id, is_representative=False) - for member in iter_mappings(entry.get("member")) - for mem_db in iter_mappings(member.get("dbReference")) - ), - ] - return { - "entity": [entity], - "cluster": [cluster], - "clustermember": [c for c in cluster_members if c], - } + def _parse() -> dict[str, list[dict[str, Any]]]: + cluster, entity = extract_cluster(entry) + cluster_id = cluster.get("cluster_id") + if not cluster_id: + logger.error("No cluster ID found in entry") + return reject_cdm_entry(entry, "No cluster ID found in entry", ctx) + + entity["updated"] = timestamp + cluster["protocol"] = uniref_variant + + cluster_members = [ + *( + extract_cross_refs(repr_db, cluster_id, is_representative=True) + for repr_member in iter_mappings(entry.get("representativeMember")) + for repr_db in iter_mappings(repr_member.get("dbReference")) + ), + *( + extract_cross_refs(mem_db, cluster_id, is_representative=False) + for member in iter_mappings(entry.get("member")) + for mem_db in iter_mappings(member.get("dbReference")) + ), + ] + + return { + "entity": [entity], + "cluster": [cluster], + "clustermember": [c for c in cluster_members if c], + } + + return run_cdm_entry_parser( + entry, + _parse, + ctx=ctx, + logger=logger, + error_message="Parse error while processing UniRef entry", + ) def make_uniref_cdm_transform( @@ -219,9 +231,12 @@ def make_uniref_cdm_transform( :return: ``RecordTransformFn`` transforming ``entry`` records into CDM table rows :rtype: RecordTransformFn """ - resolved_ts = timestamp if timestamp is not None else dt.datetime.now(dt.UTC) - return functools.partial( - parse_uniref_entry, - timestamp=resolved_ts, - uniref_variant=uniref_variant, - ) + resolved_ts = resolve_timestamp(timestamp) + + def _parse_entry( + entry: Mapping[str, Any], + ctx: TransformContext | None, + ) -> dict[str, list[dict[str, Any]]]: + return parse_uniref_entry(entry, resolved_ts, uniref_variant, ctx=ctx) + + return make_cdm_entry_transform(_parse_entry) diff --git a/src/cdm_data_loaders/io/async_client.py b/src/cdm_data_loaders/io/async_client.py index 9317c830..8464c3bc 100644 --- a/src/cdm_data_loaders/io/async_client.py +++ b/src/cdm_data_loaders/io/async_client.py @@ -1,36 +1,18 @@ -"""Asynchronous file download client. - -Usage: - -# initialise a download client that downloads files in chunks of 1024 bytes and retries the downlooad -# three times in case of connection/timeout errors -async_client = AsyncFileDownloader(max_attempts=3, chunk_size=1024) - -# output_path will be the path to the saved file -output_path = await async_client.download( - "https://example.com/file.txt", - Path("/path") / "to" / "save" / "file.txt"), - expected_checksum="some_checksum", - checksum_fn="sha512", - extra_headers={ - "If-None-Match": "abc", - "If-Modified-Since": "Wed, 21 Oct 2015 07:28:00 GMT" - }) - -""" +"""Asynchronous HTTP file download client.""" import asyncio import contextlib from logging import Logger, getLogger from pathlib import Path -from typing import Any +from typing import Any, Unpack import httpx from tenacity import AsyncRetrying from cdm_data_loaders.io.core import ( + DEFAULT_CHUNK_SIZE, HTTP_CLIENT_DEFAULTS, - RETRY_DEFAULTS, + AsyncDownloaderOptions, BaseFileDownloader, DownloadCore, ) @@ -39,85 +21,100 @@ def get_async_httpx_client() -> httpx.AsyncClient: - """Get a basic client for executing http requests.""" + """Build an asynchronous :class:`httpx.AsyncClient` configured with ``HTTP_CLIENT_DEFAULTS``. + + :return: configured async HTTP client + :rtype: httpx.AsyncClient + """ return httpx.AsyncClient(**HTTP_CLIENT_DEFAULTS) -class AsyncFileDownloader(BaseFileDownloader): - """Asynchronous downloader interface.""" +class AsyncFileDownloader(BaseFileDownloader[httpx.AsyncClient]): + """Asynchronous HTTP downloader with retries, concurrency limits, and checksum verification.""" - def __init__( # noqa: PLR0913 + def __init__( self, client: httpx.AsyncClient | None = None, - max_attempts: int = RETRY_DEFAULTS["max_attempts"], - min_backoff: int = RETRY_DEFAULTS["min_backoff"], - max_backoff: int = RETRY_DEFAULTS["max_backoff"], - chunk_size: int = 8192, - max_concurrency: int | None = None, + **options: Unpack[AsyncDownloaderOptions], ) -> None: - """Initialise an async download client. - - :param client: an httpx.Client object, defaults to None - :type client: httpx.Client | None, optional - :param max_attempts: how many times to retry the request defaults to 5 - :type max_attempts: int, optional - :param min_backoff: minimum backoff for retries, defaults to 1 - :type min_backoff: int, optional - :param max_backoff: maximum backoff for retries, defaults to 30 - :type max_backoff: int, optional - :param chunk_size: chunk size for the downloader, defaults to 8192 - :type chunk_size: int, optional - :param max_concurrency: maximum concurrency for the downloader, defaults to None - :type max_concurrency: int, optional + """Initialize an asynchronous download client. + + :param client: optional pre-configured :class:`httpx.AsyncClient` + :type client: httpx.AsyncClient | None + :param options: optional ``max_attempts``, ``min_backoff``, ``max_backoff``, ``chunk_size``, and ``max_concurrency`` + :type options: Unpack[AsyncDownloaderOptions] """ - super().__init__(client or get_async_httpx_client(), chunk_size=chunk_size) + super().__init__( + client or get_async_httpx_client(), + chunk_size=options.get("chunk_size", DEFAULT_CHUNK_SIZE), + ) + max_concurrency = options.get("max_concurrency") self.semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else None - - self._retry = AsyncRetrying(**DownloadCore.retry_kwargs(max_attempts, min_backoff, max_backoff)) + self._retry = AsyncRetrying(**self._build_retry_kwargs(options, logger)) async def download( self, url: str, destination: str | Path, expected_checksum: str | None = None, - checksum_fn: str | None = "sha256", + checksum_fn: str | None = None, extra_headers: dict[str, Any] | None = None, ) -> Path | None: """Download a file from ``url`` to ``destination`` on disk. - Returns the file path if successful; None otherwise. - 304 / Not Modified responses return None. + Returns the saved file path on 2xx responses, or ``None`` on HTTP 304 Not Modified. :param url: URL to download from :type url: str - :param destination: where to save the file to + :param destination: local destination path :type destination: str | Path - :param expected_checksum: expected checksum (if known), defaults to None - :type expected_checksum: str | None, optional - :param checksum_fn: function used for calculating the checksum, defaults to "sha256" - :type checksum_fn: str, optional - :param extra_headers: allow extra headers to be passed, defaults to None - :type extra_headers: dict[str, Any] | None, optional - :return: either the path to the downloaded file or None if no file was downloaded + :param expected_checksum: expected checksum if known + :type expected_checksum: str | None + :param checksum_fn: checksum algorithm name (defaults to ``"sha256"`` when omitted) + :type checksum_fn: str | None + :param extra_headers: optional extra request headers + :type extra_headers: dict[str, Any] | None + :return: saved file path, or ``None`` on HTTP 304 :rtype: Path | None """ - destination, checksum_fn, extra_headers = DownloadCore.validate_args(destination, checksum_fn, extra_headers) + dest_path, validated_fn, headers = DownloadCore.validate_args(destination, checksum_fn, extra_headers) async with self.semaphore or contextlib.nullcontext(): return await self._download_with_retry( url, - destination, + dest_path, expected_checksum, - checksum_fn, - extra_headers, + validated_fn, + headers, ) - async def _download_with_retry(self, *args) -> Path | None: + async def _download_with_retry( + self, + url: str, + destination: Path, + expected_checksum: str | None, + checksum_fn: str, + extra_headers: dict[str, Any], + ) -> Path | None: + """Run :meth:`_download_once` inside the configured ``AsyncRetrying`` loop. + + :param url: URL to download from + :type url: str + :param destination: local destination path + :type destination: Path + :param expected_checksum: expected checksum if known + :type expected_checksum: str | None + :param checksum_fn: validated checksum algorithm name + :type checksum_fn: str + :param extra_headers: request headers + :type extra_headers: dict[str, Any] + :return: saved file path, or ``None`` on HTTP 304 + :rtype: Path | None + :raises RuntimeError: if the retry iterator yields zero attempts + """ async for attempt in self._retry: with attempt: - return await self._download_once(*args) - # this should never be raised under normal circumstances - # tenacity should never exhaust silently + return await self._download_once(url, destination, expected_checksum, checksum_fn, extra_headers) msg = "Iterator exhausted, unreachable code reached!" raise RuntimeError(msg) @@ -129,30 +126,27 @@ async def _download_once( checksum_fn: str, extra_headers: dict[str, Any], ) -> Path | None: - """Core download function. + """Execute a single asynchronous download attempt. :param url: URL to download from :type url: str - :param destination: where to save the file to + :param destination: local destination path :type destination: Path - :param expected_checksum: expected checksum (if known), defaults to None - :type expected_checksum: str | None, optional - :param checksum_fn: function used for calculating the checksum + :param expected_checksum: expected checksum if known + :type expected_checksum: str | None + :param checksum_fn: validated checksum algorithm name :type checksum_fn: str - :param extra_headers: any extra headers to pass to the request + :param extra_headers: request headers :type extra_headers: dict[str, Any] - :raises DownloadError: for unrecoverable download errors - :raises ChecksumMismatchError: if the checksum does not match expected_checksum - :return: either the path to the downloaded file or None if no file was downloaded + :return: saved file path, or ``None`` on HTTP 304 :rtype: Path | None """ - try: + with DownloadCore.capture_download_errors(url, logger): async with self.client.stream("GET", url, headers=extra_headers) as response: if not DownloadCore.check_should_download(response, url, logger): return None chunks = response.aiter_bytes(chunk_size=self.chunk_size) - await DownloadCore.write_and_hash_async( url, destination, @@ -161,7 +155,4 @@ async def _download_once( checksum_fn=checksum_fn, ) - except Exception as exc: # noqa: BLE001 - DownloadCore.handle_download_exception(exc, url, logger) - return DownloadCore.log_download_success(url, destination, logger) diff --git a/src/cdm_data_loaders/io/core.py b/src/cdm_data_loaders/io/core.py index 57a7527b..526107eb 100644 --- a/src/cdm_data_loaders/io/core.py +++ b/src/cdm_data_loaders/io/core.py @@ -1,37 +1,57 @@ -"""File downloader with tenacity for retry support.""" +"""Shared retry, response validation, and streaming checksum logic for HTTP downloaders.""" from collections.abc import AsyncIterable, Callable, Generator, Iterable from contextlib import contextmanager -from logging import Logger, getLogger +from http import HTTPStatus +from logging import WARNING, Logger, getLogger from pathlib import Path -from typing import Any, Never +from typing import Any, Final, TypedDict import httpx from frozendict import frozendict from tenacity import ( + before_sleep_log, retry_if_exception_type, stop_after_attempt, wait_exponential, ) -from cdm_data_loaders.utils.checksums import new_hasher, validate_checksum_fn +from cdm_data_loaders.utils.checksums import Hasher, new_hasher, validate_checksum_fn logger: Logger = getLogger(__name__) +DEFAULT_CHECKSUM_FN: Final[str] = "sha256" +DEFAULT_CHUNK_SIZE: Final[int] = 8192 + class DownloadError(Exception): """Retryable download error.""" class NonRetryableDownloadError(Exception): - """Retryable download error.""" + """Non-retryable client or validation error.""" class ChecksumMismatchError(NonRetryableDownloadError): """Checksum validation failed.""" -HTTP_CLIENT_DEFAULTS = frozendict( +class DownloaderOptions(TypedDict, total=False): + """Optional retry and streaming parameters for HTTP file downloaders.""" + + max_attempts: int + min_backoff: int + max_backoff: int + chunk_size: int + + +class AsyncDownloaderOptions(DownloaderOptions, total=False): + """Optional retry, streaming, and concurrency parameters for :class:`AsyncFileDownloader`.""" + + max_concurrency: int | None + + +HTTP_CLIENT_DEFAULTS: Final[frozendict[str, Any]] = frozendict( { "timeout": httpx.Timeout(30.0), "limits": httpx.Limits( @@ -43,7 +63,7 @@ class ChecksumMismatchError(NonRetryableDownloadError): ) -RETRY_DEFAULTS = frozendict( +RETRY_DEFAULTS: Final[frozendict[str, int]] = frozendict( { "max_attempts": 5, "min_backoff": 1, @@ -53,32 +73,61 @@ class ChecksumMismatchError(NonRetryableDownloadError): class DownloadCore: - """HTTP file downloader with retries and checksum testing.""" + """HTTP file downloader helpers for retries, status checks, and streaming checksums.""" - RETRYABLE_EXCEPTIONS = ( + RETRYABLE_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = ( httpx.TimeoutException, httpx.TransportError, DownloadError, ) @staticmethod - def retry_kwargs(max_attempts: int, min_backoff: int, max_backoff: int) -> dict[str, Any]: - """Build common tenacity retry configuration for sync and async downloaders.""" + def retry_kwargs(max_attempts: int, min_backoff: int, max_backoff: int, log: Logger) -> dict[str, Any]: + """Build tenacity retry configuration shared by sync and async downloaders. + + :param max_attempts: maximum attempts before re-raising + :type max_attempts: int + :param min_backoff: minimum exponential backoff in seconds + :type min_backoff: int + :param max_backoff: maximum exponential backoff in seconds + :type max_backoff: int + :param log: logger used for pre-sleep retry warnings + :type log: Logger + :return: dictionary of keyword arguments for ``tenacity.retry`` or ``AsyncRetrying`` + :rtype: dict[str, Any] + """ return { "retry": retry_if_exception_type(DownloadCore.RETRYABLE_EXCEPTIONS), "stop": stop_after_attempt(max_attempts), "wait": wait_exponential(min=min_backoff, max=max_backoff), "reraise": True, + "before_sleep": before_sleep_log(log, WARNING), } @staticmethod - def handle_download_exception(exc: Exception, url: str, log: Logger) -> Never: - """Log a download failure and re-raise or wrap it in ``DownloadError``.""" - if isinstance(exc, NonRetryableDownloadError): - log.exception("%s: %s; retry not possible", url, exc.args[0], extra={"url": url}) # noqa: LOG004 - raise exc - log.exception("%s: %s; retry possible", url, exc.args[0], extra={"url": url}) # noqa: LOG004 - raise DownloadError(str(exc)) from exc + @contextmanager + def capture_download_errors(url: str, log: Logger) -> Generator[None]: + """Translate transport and HTTP errors into logged :class:`DownloadError` or :class:`NonRetryableDownloadError`. + + :param url: target URL + :type url: str + :param log: logger instance + :type log: Logger + :yield: ``None`` + :rtype: Generator[None] + :raises NonRetryableDownloadError: when a non-retryable client or checksum error occurs + :raises DownloadError: when a retryable transport, timeout, or server error occurs + """ + try: + yield + except NonRetryableDownloadError as exc: + detail = exc.args[0] if exc.args else str(exc) + log.exception("%s: %s; retry not possible", url, detail, extra={"url": url}) + raise + except Exception as exc: + detail = exc.args[0] if exc.args else str(exc) + log.exception("%s: %s; retry possible", url, detail, extra={"url": url}) + raise DownloadError(str(exc)) from exc @staticmethod def validate_args( @@ -86,50 +135,45 @@ def validate_args( checksum_fn: str | None = None, extra_headers: dict[str, Any] | None = None, ) -> tuple[Path, str, dict[str, Any]]: - """Validate download arguments, apply defaults, and convert them to the appropriate format. - - If no checksum function is supplied, it will be set to "sha256". + """Validate download arguments, apply defaults, and ensure parent directories exist. - :param destination: location to save file to + :param destination: location to save the downloaded file :type destination: str | Path - :param checksum_fn: function for calculating file checksum - :type checksum_fn: str | None, optional - :param extra_headers: extra headers to add to the httpx request, defaults to None - :type extra_headers: dict[str, Any] | None, optional - :return: arguments spruced up and in shape! + :param checksum_fn: checksum algorithm name (defaults to ``"sha256"`` when omitted) + :type checksum_fn: str | None + :param extra_headers: optional extra HTTP request headers + :type extra_headers: dict[str, Any] | None + :return: tuple of ``(destination_path, validated_checksum_fn, extra_headers)`` :rtype: tuple[Path, str, dict[str, Any]] """ - if not checksum_fn: - checksum_fn = "sha256" - - validated_checksum_fn = validate_checksum_fn(checksum_fn, include_extras=False) - destination = Path(destination) - destination.parent.mkdir(parents=True, exist_ok=True) - - if not extra_headers: - extra_headers = {} - - return (destination, validated_checksum_fn, extra_headers) + validated_checksum_fn = validate_checksum_fn(checksum_fn or DEFAULT_CHECKSUM_FN, include_extras=False) + destination_path = Path(destination) + destination_path.parent.mkdir(parents=True, exist_ok=True) + return (destination_path, validated_checksum_fn, extra_headers or {}) @staticmethod def validate_response(response: httpx.Response) -> bool: - """Check the response from the server and act accordingly.""" + """Validate an HTTP response status code. + + :param response: HTTP response from the server + :type response: httpx.Response + :return: ``True`` on 2xx, ``False`` on 304 Not Modified + :rtype: bool + :raises NonRetryableDownloadError: on 4xx client errors + :raises DownloadError: on unexpected 3xx or 5xx responses + """ status = response.status_code - # not modified since last accessed - if status == 304: # noqa: PLR2004 + if status == HTTPStatus.NOT_MODIFIED: return False - # successful response - if 200 <= status < 300: # noqa: PLR2004 + if HTTPStatus.OK <= status < HTTPStatus.MULTIPLE_CHOICES: return True - # client error - if 400 <= status < 500: # noqa: PLR2004 + if HTTPStatus.BAD_REQUEST <= status < HTTPStatus.INTERNAL_SERVER_ERROR: msg = f"Client error: {status} {response.reason_phrase}" raise NonRetryableDownloadError(msg) - # server errors msg = f"Server error: {status} {response.reason_phrase}" raise DownloadError(msg) @@ -153,7 +197,7 @@ def check_should_download(response: httpx.Response, url: str, log: Logger) -> bo @staticmethod def log_download_success(url: str, destination: Path, log: Logger) -> Path: - """Log a successful download and return ``destination``. + """Log a completed download and return ``destination``. :param url: download URL :type url: str @@ -172,22 +216,21 @@ def check_hash( url: str, expected_checksum: str, checksum_fn: str, - hash_obj, #: hashlib._Hash, + hash_obj: Hasher | None, ) -> None: - """Compare the expected to actual checksum. + """Compare the computed digest against ``expected_checksum``. :param url: download URL :type url: str - :param expected_checksum: expected checksum + :param expected_checksum: expected hexadecimal digest :type expected_checksum: str - :param checksum_fn: checksum algorithm + :param checksum_fn: checksum algorithm name :type checksum_fn: str - :param hash_obj: hash object for calculating checksum - :type hash_obj: bool - :raises ChecksumMismatchError: if the checksum is incorrect + :param hash_obj: populated hasher instance, or ``None`` when hashing was skipped + :type hash_obj: Hasher | None + :raises ChecksumMismatchError: if the digest does not match ``expected_checksum`` """ - if hash_obj: - # check the checksum matches expectations + if hash_obj is not None: actual = hash_obj.hexdigest() if actual.lower() != expected_checksum.lower(): msg = f"{url}: Checksum mismatch: expected={expected_checksum}, actual={actual}" @@ -220,12 +263,12 @@ def open_hash_writer( def _write_chunk(chunk: bytes) -> None: fh.write(chunk) - if hash_obj: + if hash_obj is not None: hash_obj.update(chunk) yield _write_chunk - if hash_obj and expected_checksum: + if hash_obj is not None and expected_checksum: DownloadCore.check_hash(url, expected_checksum, checksum_fn, hash_obj) @staticmethod @@ -236,19 +279,19 @@ def write_and_hash( expected_checksum: str | None, checksum_fn: str, ) -> None: - """Write downloaded data to disk and accrue data for calculating the checksum of the download. + """Write downloaded byte chunks to ``destination`` and verify ``expected_checksum`` if supplied. :param url: URL to download from :type url: str - :param destination: where to save the file to + :param destination: file path to save to :type destination: Path :param chunks: iterable chunks of downloaded data :type chunks: Iterable[bytes] - :param expected_checksum: expected checksum (if known), defaults to None - :type expected_checksum: str | None, optional - :param checksum_fn: function used for calculating the checksum, defaults to "sha256" - :type checksum_fn: str, optional - :raises ChecksumMismatchError: if the checksum of the downloaded data does not match the expected checksum. + :param expected_checksum: expected checksum if known + :type expected_checksum: str | None + :param checksum_fn: checksum algorithm name + :type checksum_fn: str + :raises ChecksumMismatchError: if the computed checksum does not match ``expected_checksum`` """ with DownloadCore.open_hash_writer(url, destination, expected_checksum, checksum_fn) as write_chunk: for chunk in chunks: @@ -262,37 +305,55 @@ async def write_and_hash_async( expected_checksum: str | None, checksum_fn: str, ) -> None: - """Write downloaded data to disk and accrue data for calculating the checksum of the download. + """Write async byte chunks to ``destination`` and verify ``expected_checksum`` if supplied. :param url: URL to download from :type url: str - :param destination: where to save the file to + :param destination: file path to save to :type destination: Path - :param chunks: iterable chunks of downloaded data + :param chunks: async iterable chunks of downloaded data :type chunks: AsyncIterable[bytes] - :param expected_checksum: expected checksum (if known), defaults to None - :type expected_checksum: str | None, optional - :param checksum_fn: function used for calculating the checksum, defaults to "sha256" - :type checksum_fn: str, optional - :raises ChecksumMismatchError: if the checksum of the downloaded data does not match the expected checksum. + :param expected_checksum: expected checksum if known + :type expected_checksum: str | None + :param checksum_fn: checksum algorithm name + :type checksum_fn: str + :raises ChecksumMismatchError: if the computed checksum does not match ``expected_checksum`` """ with DownloadCore.open_hash_writer(url, destination, expected_checksum, checksum_fn) as write_chunk: async for chunk in chunks: write_chunk(chunk) -class BaseFileDownloader: +class BaseFileDownloader[ClientT: (httpx.Client, httpx.AsyncClient)]: """Shared base class for synchronous and asynchronous HTTP file downloaders.""" RETRYABLE_EXCEPTIONS = DownloadCore.RETRYABLE_EXCEPTIONS - def __init__(self, client: Any, *, chunk_size: int = 8192) -> None: # noqa: ANN401 + def __init__(self, client: ClientT, *, chunk_size: int = DEFAULT_CHUNK_SIZE) -> None: """Initialize shared HTTP client and chunk size attributes. :param client: configured sync or async httpx client - :type client: Any + :type client: ClientT :param chunk_size: download stream chunk size in bytes :type chunk_size: int """ self.client = client self.chunk_size = chunk_size + + @staticmethod + def _build_retry_kwargs(options: DownloaderOptions, log: Logger) -> dict[str, Any]: + """Extract retry configuration from ``options`` with defaults from ``RETRY_DEFAULTS``. + + :param options: downloader option mapping + :type options: DownloaderOptions + :param log: logger used for pre-sleep retry warnings + :type log: Logger + :return: keyword arguments for ``tenacity.retry`` or ``AsyncRetrying`` + :rtype: dict[str, Any] + """ + return DownloadCore.retry_kwargs( + options.get("max_attempts", RETRY_DEFAULTS["max_attempts"]), + options.get("min_backoff", RETRY_DEFAULTS["min_backoff"]), + options.get("max_backoff", RETRY_DEFAULTS["max_backoff"]), + log, + ) diff --git a/src/cdm_data_loaders/io/ftp_client.py b/src/cdm_data_loaders/io/ftp_client.py index 1704b6dd..f4aa2d67 100644 --- a/src/cdm_data_loaders/io/ftp_client.py +++ b/src/cdm_data_loaders/io/ftp_client.py @@ -14,7 +14,7 @@ from ftplib import FTP, error_temp from logging import Logger, getLogger from pathlib import Path, PurePosixPath -from typing import Any +from typing import Any, Final from tenacity import ( before_sleep_log, @@ -24,9 +24,19 @@ wait_fixed, ) +from cdm_data_loaders.utils.checksums import finalize_digest, new_hasher, validate_checksum_fn + logger: Logger = getLogger(__name__) -DEFAULT_TIMEOUT = 60 +DEFAULT_TIMEOUT: Final[int] = 60 +DEFAULT_RETRIES: Final[int] = 3 +RETRY_WAIT_SECONDS: Final[int] = 2 +TRANSIENT_FTP_EXCEPTIONS: Final[tuple[type[BaseException], ...]] = ( + error_temp, + BrokenPipeError, + ConnectionResetError, + EOFError, +) def connect_ftp(host: str, timeout: int = DEFAULT_TIMEOUT) -> FTP: @@ -103,17 +113,23 @@ def ftp_noop_keepalive(ftp: FTP, last_activity: float, interval: int = 25) -> fl def _ftp_retry(retries: int) -> Callable[[Callable[..., Any]], Callable[..., Any]]: - """Build a tenacity retry decorator for transient FTP errors.""" + """Build a tenacity retry decorator for transient FTP and socket errors. + + :param retries: total attempt count before re-raising + :type retries: int + :return: tenacity retry decorator + :rtype: Callable[[Callable[..., Any]], Callable[..., Any]] + """ return retry( - retry=retry_if_exception_type(error_temp), + retry=retry_if_exception_type(TRANSIENT_FTP_EXCEPTIONS), stop=stop_after_attempt(retries), - wait=wait_fixed(2), + wait=wait_fixed(RETRY_WAIT_SECONDS), reraise=True, before_sleep=before_sleep_log(logger, logging.WARNING), ) -def ftp_list_dir(ftp: FTP, path: PurePosixPath, retries: int = 3) -> list[PurePosixPath]: +def ftp_list_dir(ftp: FTP, path: PurePosixPath, retries: int = DEFAULT_RETRIES) -> list[PurePosixPath]: """List files in an FTP directory with retry on transient errors. :param ftp: active FTP connection @@ -123,6 +139,7 @@ def ftp_list_dir(ftp: FTP, path: PurePosixPath, retries: int = 3) -> list[PurePo :param retries: number of retry attempts :type retries: int :return: list of filenames + :rtype: list[PurePosixPath] """ ftp.cwd(str(path)) @@ -135,39 +152,67 @@ def _list() -> list[PurePosixPath]: return _list() -def ftp_download_file(ftp: FTP, remote_path: PurePosixPath, local_path: Path, retries: int = 3) -> None: - """Download a single file from FTP with retry on transient errors. +def ftp_download_file( + ftp: FTP, + remote_path: PurePosixPath, + local_path: Path, + retries: int = DEFAULT_RETRIES, + *, + checksum_fn: str | None = None, +) -> str | None: + """Download a single file from FTP with retry on transient errors and optional in-stream hashing. :param ftp: active FTP connection :type ftp: FTP - :param remote_path: full remote file path + :param remote_path: remote file path :type remote_path: PurePosixPath :param local_path: local destination path :type local_path: Path :param retries: number of retry attempts :type retries: int + :param checksum_fn: optional checksum algorithm name (for example, ``"md5"``) + :type checksum_fn: str | None + :return: computed checksum digest when ``checksum_fn`` is set, otherwise ``None`` + :rtype: str | None """ + validated_algo = validate_checksum_fn(checksum_fn) if checksum_fn is not None else None @_ftp_retry(retries) - def _download() -> None: + def _download() -> str | None: + hasher = new_hasher(validated_algo) if validated_algo is not None else None with local_path.open("wb") as f: - ftp.retrbinary(f"RETR {remote_path}", f.write) - _download() + def _write_chunk(chunk: bytes) -> None: + f.write(chunk) + if hasher is not None: + hasher.update(chunk) + + ftp.retrbinary(f"RETR {remote_path}", _write_chunk) + return finalize_digest(hasher) if hasher is not None else None + return _download() -def ftp_retrieve_text(ftp: FTP, remote_path: PurePosixPath) -> str: - """Retrieve a text file from FTP, returning its content as a string. + +def ftp_retrieve_text(ftp: FTP, remote_path: PurePosixPath, retries: int = DEFAULT_RETRIES) -> str: + """Retrieve a text file from FTP with retry on transient errors. :param ftp: active FTP connection :type ftp: FTP - :param remote_path: full remote file path + :param remote_path: remote file path :type remote_path: PurePosixPath - :return: file content + :param retries: number of retry attempts + :type retries: int + :return: file content joined with newlines + :rtype: str """ - lines: list[str] = [] - ftp.retrlines(f"RETR {remote_path}", lines.append) - return "\n".join(lines) + + @_ftp_retry(retries) + def _retrieve() -> str: + lines: list[str] = [] + ftp.retrlines(f"RETR {remote_path}", lines.append) + return "\n".join(lines) + + return _retrieve() class ThreadLocalFTP: diff --git a/src/cdm_data_loaders/io/s3/object_utils.py b/src/cdm_data_loaders/io/s3/object_utils.py index af3126d2..d476b536 100644 --- a/src/cdm_data_loaders/io/s3/object_utils.py +++ b/src/cdm_data_loaders/io/s3/object_utils.py @@ -3,6 +3,7 @@ import itertools import shutil from collections.abc import Callable +from http import HTTPStatus from logging import Logger, getLogger from pathlib import Path from typing import IO, Any, Final @@ -72,27 +73,36 @@ def head_object(s3_path: str) -> dict[str, Any]: return s3.head_object(Bucket=bucket, Key=key, ChecksumMode="ENABLED") # type: ignore[call-arg] +def is_not_found_client_error(exc: ClientError) -> bool: + """Return ``True`` when ``exc`` represents an S3 object-or-bucket not-found error. + + :param exc: botocore ``ClientError`` raised by an S3 operation + :type exc: ClientError + :return: ``True`` if the error code is in ``NOT_FOUND_ERROR_CODES`` or HTTP status is 404 + :rtype: bool + """ + error_code = exc.response.get("Error", {}).get("Code") + status_code = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") + return error_code in NOT_FOUND_ERROR_CODES or status_code == HTTPStatus.NOT_FOUND + + def get_existing_object_info(s3_path: str) -> dict[str, Any] | None: - """Fetch metadata for an object already in S3, if it exists. + """Fetch metadata for an object in S3 if it exists. - :param s3_path: path to the object on s3, including the bucket name + :param s3_path: path to the object on S3, including the bucket name :type s3_path: str - :raises ClientError: for any S3 error other than "object not found" - :return: metadata for the existing object, or None if no object exists at `bucket`/`key` + :raises ClientError: for any S3 error other than object/bucket not found + :return: metadata for the object, or ``None`` if no object exists at ``s3_path`` :rtype: dict[str, Any] | None """ try: response = head_object(s3_path) except ClientError as exc: - # AWS service errors, client-side issues - logger.exception("Error checking for existing object: %s", s3_path) - error_code = exc.response.get("Error", {}).get("Code") - status_code = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode") - if error_code in NOT_FOUND_ERROR_CODES or status_code == 404: # noqa: PLR2004 + if is_not_found_client_error(exc): return None + logger.exception("Error checking for existing object: %s", s3_path) raise except Exception: - # any other errors - e.g. invalid/missing args logger.exception("Error checking for existing object: %s", s3_path) raise @@ -124,6 +134,24 @@ def get_object_bytes(s3_path: str) -> bytes: return resp["Body"].read() +def get_object_bytes_if_exists(s3_path: str) -> bytes | None: + """Fetch an S3 object's byte payload, returning ``None`` when the object is absent. + + :param s3_path: path to the object on S3, including the bucket name + :type s3_path: str + :raises ClientError: for any S3 error other than object/bucket not found + :return: object payload bytes, or ``None`` if not found + :rtype: bytes | None + """ + try: + return get_object_bytes(s3_path) + except ClientError as exc: + if is_not_found_client_error(exc): + return None + logger.exception("Error reading object: %s", s3_path) + raise + + def put_object_bytes( data: bytes, s3_path: str, diff --git a/src/cdm_data_loaders/io/sync_client.py b/src/cdm_data_loaders/io/sync_client.py index fc62e306..1ddd0dcb 100644 --- a/src/cdm_data_loaders/io/sync_client.py +++ b/src/cdm_data_loaders/io/sync_client.py @@ -1,79 +1,108 @@ -"""Synchronous file download client. +"""Synchronous HTTP file and JSON download client.""" -Usage: - -# initialise a download client that downloads files in chunks of 1024 bytes and retries the downlooad -# three times in case of connection/timeout errors -sync_client = FileDownloader(max_attempts=3, chunk_size=1024) - -# output_path will be the path to the saved file -output_path = sync_client.download( - "https://example.com/file.txt", - Path("/path") / "to" / "save" / "file.txt"), - expected_checksum="some_checksum", - checksum_fn="sha512", - extra_headers={ - "If-None-Match": "abc", - "If-Modified-Since": "Wed, 21 Oct 2015 07:28:00 GMT" - }) - -""" - -from logging import WARNING, Logger, getLogger +from logging import Logger, getLogger from pathlib import Path -from typing import Any +from typing import Any, Unpack import httpx -from tenacity import ( - before_sleep_log, - retry, -) +from tenacity import retry from cdm_data_loaders.io.core import ( + DEFAULT_CHUNK_SIZE, HTTP_CLIENT_DEFAULTS, - RETRY_DEFAULTS, BaseFileDownloader, DownloadCore, + DownloaderOptions, ) logger: Logger = getLogger(__name__) def get_httpx_client() -> httpx.Client: - """Get a basic client for executing http requests.""" + """Build a synchronous :class:`httpx.Client` configured with ``HTTP_CLIENT_DEFAULTS``. + + :return: configured HTTP client + :rtype: httpx.Client + """ return httpx.Client(**HTTP_CLIENT_DEFAULTS) -class FileDownloader(BaseFileDownloader): - """Synchronous downloader interface.""" +def fetch_json( + url: str, + *, + timeout: float | None = None, + extra_headers: dict[str, str] | None = None, + **options: Unpack[DownloaderOptions], +) -> dict[str, Any]: + """Fetch and decode a JSON object from ``url`` using a managed :class:`httpx.Client` with retries. + + :param url: URL to request + :type url: str + :param timeout: optional request timeout in seconds overriding ``HTTP_CLIENT_DEFAULTS`` + :type timeout: float | None + :param extra_headers: optional HTTP request headers + :type extra_headers: dict[str, str] | None + :param options: optional retry settings forwarded to :class:`FileDownloader` + :type options: Unpack[DownloaderOptions] + :return: decoded JSON dictionary + :rtype: dict[str, Any] + """ + client_kwargs: dict[str, Any] = dict(HTTP_CLIENT_DEFAULTS) + if timeout is not None: + client_kwargs["timeout"] = httpx.Timeout(timeout) + with httpx.Client(**client_kwargs) as http_client: + return FileDownloader(client=http_client, **options).fetch_json(url, extra_headers=extra_headers) + + +class FileDownloader(BaseFileDownloader[httpx.Client]): + """Synchronous HTTP downloader with retries and streaming checksum verification.""" def __init__( self, client: httpx.Client | None = None, - max_attempts: int = RETRY_DEFAULTS["max_attempts"], - min_backoff: int = RETRY_DEFAULTS["min_backoff"], - max_backoff: int = RETRY_DEFAULTS["max_backoff"], - chunk_size: int = 8192, + **options: Unpack[DownloaderOptions], ) -> None: - """Initialise a synchronous download client. - - :param client: an httpx.Client object, defaults to None - :type client: httpx.Client | None, optional - :param max_attempts: how many times to retry the request defaults to 5 - :type max_attempts: int, optional - :param min_backoff: minimum backoff for retries, defaults to 1 - :type min_backoff: int, optional - :param max_backoff: maximum backoff for retries, defaults to 30 - :type max_backoff: int, optional - :param chunk_size: chunk size for the downloader, defaults to 8192 - :type chunk_size: int, optional - """ - super().__init__(client or get_httpx_client(), chunk_size=chunk_size) + """Initialize a synchronous download client. - self._retry = retry( - **DownloadCore.retry_kwargs(max_attempts, min_backoff, max_backoff), - before_sleep=before_sleep_log(logger, WARNING), + :param client: optional pre-configured :class:`httpx.Client` + :type client: httpx.Client | None + :param options: optional ``max_attempts``, ``min_backoff``, ``max_backoff``, and ``chunk_size`` + :type options: Unpack[DownloaderOptions] + """ + super().__init__( + client or get_httpx_client(), + chunk_size=options.get("chunk_size", DEFAULT_CHUNK_SIZE), ) + self._retry = retry(**self._build_retry_kwargs(options, logger)) + + def fetch_json( + self, + url: str, + *, + extra_headers: dict[str, str] | None = None, + ) -> dict[str, Any]: + """Fetch and decode a JSON object from ``url`` with retry on transient errors. + + :param url: URL to request + :type url: str + :param extra_headers: optional request headers + :type extra_headers: dict[str, str] | None + :return: decoded JSON payload + :rtype: dict[str, Any] + :raises DownloadError: after exhausting retries on transport, timeout, or 5xx errors + :raises NonRetryableDownloadError: on 4xx client errors + """ + headers = extra_headers or {} + + @self._retry + def _once() -> dict[str, Any]: + with DownloadCore.capture_download_errors(url, logger): + response = self.client.get(url, headers=headers) + DownloadCore.validate_response(response) + payload: dict[str, Any] = response.json() + return payload + + return _once() def download( self, @@ -85,27 +114,26 @@ def download( ) -> Path | None: """Download a file from ``url`` to ``destination`` on disk. - Returns the file path if successful; None otherwise. - 304 / Not Modified responses return None. + Returns the saved file path on 2xx responses, or ``None`` on HTTP 304 Not Modified. :param url: URL to download from :type url: str - :param destination: where to save the file to + :param destination: local destination path :type destination: str | Path - :param expected_checksum: expected checksum (if known), defaults to None - :type expected_checksum: str | None, optional - :param checksum_fn: function used for calculating the checksum. Set to "sha256" if not supplied. - :type checksum_fn: str, optional - :param extra_headers: allow extra headers to be passed, defaults to None - :type extra_headers: dict[str, Any] | None, optional - :return: either the path to the downloaded file or None if no file was downloaded + :param expected_checksum: expected checksum if known + :type expected_checksum: str | None + :param checksum_fn: checksum algorithm name (defaults to ``"sha256"`` when omitted) + :type checksum_fn: str | None + :param extra_headers: optional extra request headers + :type extra_headers: dict[str, Any] | None + :return: saved file path, or ``None`` on HTTP 304 :rtype: Path | None """ - destination, checksum_fn, extra_headers = DownloadCore.validate_args(destination, checksum_fn, extra_headers) + dest_path, validated_fn, headers = DownloadCore.validate_args(destination, checksum_fn, extra_headers) @self._retry def _once() -> Path | None: - return self._download_once(url, destination, expected_checksum, checksum_fn, extra_headers) + return self._download_once(url, dest_path, expected_checksum, validated_fn, headers) return _once() @@ -117,39 +145,35 @@ def _download_once( checksum_fn: str, extra_headers: dict[str, Any], ) -> Path | None: - """Core synchronous download function. + """Execute a single synchronous download attempt. :param url: URL to download from :type url: str - :param destination: where to save the file to + :param destination: local destination path :type destination: Path - :param expected_checksum: expected checksum (if known), defaults to None - :type expected_checksum: str | None, optional - :param checksum_fn: function used for calculating the checksum, defaults to "sha256" - :type checksum_fn: str, optional - :param extra_headers: any extra headers to pass to the request, defaults to None - :type extra_headers: dict[str, Any] | None, optional - :raises DownloadError: for unrecoverable download errors - :raises ChecksumMismatchError: if the checksum does not match expected_checksum - :return: either the path to the downloaded file or None if no file was downloaded + :param expected_checksum: expected checksum if known + :type expected_checksum: str | None + :param checksum_fn: validated checksum algorithm name + :type checksum_fn: str + :param extra_headers: request headers + :type extra_headers: dict[str, Any] + :return: saved file path, or ``None`` on HTTP 304 :rtype: Path | None """ - try: - with self.client.stream("GET", url, headers=extra_headers) as response: - if not DownloadCore.check_should_download(response, url, logger): - return None - - chunks = response.iter_bytes(chunk_size=self.chunk_size) - - DownloadCore.write_and_hash( - url, - destination, - chunks, - expected_checksum=expected_checksum, - checksum_fn=checksum_fn, - ) - - except Exception as exc: # noqa: BLE001 - DownloadCore.handle_download_exception(exc, url, logger) + with ( + DownloadCore.capture_download_errors(url, logger), + self.client.stream("GET", url, headers=extra_headers) as response, + ): + if not DownloadCore.check_should_download(response, url, logger): + return None + + chunks = response.iter_bytes(chunk_size=self.chunk_size) + DownloadCore.write_and_hash( + url, + destination, + chunks, + expected_checksum=expected_checksum, + checksum_fn=checksum_fn, + ) return DownloadCore.log_download_success(url, destination, logger) diff --git a/src/cdm_data_loaders/pipelines/jsonlines/base.py b/src/cdm_data_loaders/pipelines/jsonlines/base.py index 26b34b66..847e94ea 100644 --- a/src/cdm_data_loaders/pipelines/jsonlines/base.py +++ b/src/cdm_data_loaders/pipelines/jsonlines/base.py @@ -112,7 +112,7 @@ def run_registry_ingest( :type registry: dict[RegistryKey, type[BaseModel] | ValidatedSchema] :param build_resource: function building one entity's resource :type build_resource: ResourceFactory - :param pipeline_name: registered pipeline name + :param pipeline_name: default pipeline name when ``settings.model_config['cli_prog_name']`` is unset :type pipeline_name: str :param pipeline_run_kwargs: extra kwargs for pipeline.run :type pipeline_run_kwargs: dict[str, Any] | None diff --git a/src/cdm_data_loaders/pipelines/jsonlines/jsonschema.py b/src/cdm_data_loaders/pipelines/jsonlines/jsonschema.py index 5ec90af3..f1981919 100644 --- a/src/cdm_data_loaders/pipelines/jsonlines/jsonschema.py +++ b/src/cdm_data_loaders/pipelines/jsonlines/jsonschema.py @@ -82,12 +82,11 @@ def run_jsonlines_pipeline(settings: JsonlJsonschemaIngestSettings) -> LoadInfo :return: load information for the pipeline :rtype: LoadInfo | None """ - pipeline_name = str(settings.model_config.get("cli_prog_name") or JSONSCHEMA_PIPELINE_NAME) return run_registry_ingest( settings, registry=settings.schema_registry.registry, build_resource=build_entity_resource, - pipeline_name=pipeline_name, + pipeline_name=JSONSCHEMA_PIPELINE_NAME, ) diff --git a/src/cdm_data_loaders/pipelines/jsonlines/settings.py b/src/cdm_data_loaders/pipelines/jsonlines/settings.py index 17d7f4ba..1fba889b 100644 --- a/src/cdm_data_loaders/pipelines/jsonlines/settings.py +++ b/src/cdm_data_loaders/pipelines/jsonlines/settings.py @@ -46,7 +46,7 @@ ) from cdm_data_loaders.core.transforms.settings_mixin import ( TransformSettingsMixin, - needs_default_pyarrow_schema, + reconcile_multi_table_schema_map, validate_table_transforms, ) from cdm_data_loaders.pipelines.jsonlines.exceptions import ( @@ -124,7 +124,6 @@ def build_output_schema_registry( :rtype: ArrowSchemaRegistry | None :raises RuntimeError: if one or more schemas cannot be converted to PyArrow schemas """ - pyarrow_schemas: dict[str, pa.Schema] = dict(output_schemas.registry) if output_schemas is not None else {} errors: dict[str, str] = {} has_arrow_error = False is_model_registry = isinstance(validation_schema_registry, ModelRegistry) @@ -134,15 +133,11 @@ def build_output_schema_registry( else: fn = convert_schema_to_pyarrow - for table_name, model_or_schema in validation_schema_registry.registry.items(): - if not needs_default_pyarrow_schema( - table_name, - table_transforms=table_transforms, - output_schemas=output_schemas, - ): - continue + def _convert_table_schema(table_name: str) -> pa.Schema | None: + nonlocal has_arrow_error + model_or_schema = validation_schema_registry.registry[table_name] try: - pyarrow_schemas[table_name] = fn(model_or_schema) + return fn(model_or_schema) except PydanticToPyArrowError as error: logger.exception("Error converting Pydantic model to PyArrow for %s", table_name) errors[table_name] = str(error) @@ -153,6 +148,14 @@ def build_output_schema_registry( logger.exception("Error converting JSON Schema to PyArrow for %s", table_name) has_arrow_error = True errors[table_name] = str(error) + return None + + registry_result = reconcile_multi_table_schema_map( + validation_schema_registry.registry, + _convert_table_schema, + table_transforms=table_transforms, + output_schemas=output_schemas, + ) if errors: details = "".join(f"{table_name}: {error}\n" for table_name, error in errors.items()) @@ -163,10 +166,7 @@ def build_output_schema_registry( raise UnsupportedJsonlArrowSchemaError(err_msg) raise UnresolvableJsonlSchemaRefError(err_msg) - if not pyarrow_schemas: - return None - - return ArrowSchemaRegistry(registry=pyarrow_schemas) + return registry_result class JsonlIngestSettings(FileFormatIngestSettings): diff --git a/src/cdm_data_loaders/pipelines/presets/pubmed.py b/src/cdm_data_loaders/pipelines/presets/pubmed.py index 7ece25d1..017546c5 100644 --- a/src/cdm_data_loaders/pipelines/presets/pubmed.py +++ b/src/cdm_data_loaders/pipelines/presets/pubmed.py @@ -6,14 +6,12 @@ import pyarrow as pa from pydantic import Field, FilePath -from pydantic_settings import SettingsConfigDict from cdm_data_loaders.core.cli import CliRunner from cdm_data_loaders.core.fields import FileGlob, file_glob_description from cdm_data_loaders.core.md5_preprocessor import validate_md5_sidecar from cdm_data_loaders.core.registry import FilePreprocessFn -from cdm_data_loaders.core.settings import default_settings_with_shortcuts -from cdm_data_loaders.pipelines.xml_common import XML_CLI_SHORTCUTS, XML_GZ_FILE_GLOB +from cdm_data_loaders.pipelines.xml_common import XML_GZ_FILE_GLOB from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import run_xmltopyarrow_pipeline from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( SCHEMAS_DIR, @@ -26,17 +24,11 @@ PUBMED_XSD_FILE: Final[Path] = SCHEMAS_DIR / "pubmed" / "pubmed.xsd" -class PubMedXmlToPyArrowSettings(LevelPresetXmlToPyArrowSettings): +class PubMedXmlToPyArrowSettings(LevelPresetXmlToPyArrowSettings, cli_prog_name=PUBMED_PIPELINE_NAME): """Settings for the PubMed XML-to-PyArrow ingestion pipeline.""" PRECOMPUTED_SCHEMAS: ClassVar[Mapping[str, pa.Schema]] = PUBMED_SCHEMAS DEFAULT_FILE_PREPROCESSORS: ClassVar[tuple[FilePreprocessFn, ...]] = (validate_md5_sidecar,) - - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=PUBMED_PIPELINE_NAME, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - dataset_name: ClassVar[str] = PUBMED_PIPELINE_NAME file_glob: Annotated[ diff --git a/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py b/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py index 27b3f2db..1364641b 100644 --- a/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py +++ b/src/cdm_data_loaders/pipelines/presets/uniprot_cdm.py @@ -4,18 +4,16 @@ import pyarrow as pa from pydantic import Field -from pydantic_settings import SettingsConfigDict from cdm_data_loaders.core.cli import CliRunner from cdm_data_loaders.core.fields import LogInterval -from cdm_data_loaders.core.settings import default_settings_with_shortcuts +from cdm_data_loaders.core.transforms.settings_mixin import LockedTransformHooksMixin from cdm_data_loaders.datasets.uniprot.kb import UNIPROT_CDM_SCHEMA_MAP, make_uniprot_cdm_transform from cdm_data_loaders.datasets.uniprot.uniref import UNIREF_CDM_SCHEMA_MAP, make_uniref_cdm_transform from cdm_data_loaders.pipelines.presets.uniprot_xml import ( UniProtXmlToPyArrowSettings, UniRefXmlToPyArrowSettings, ) -from cdm_data_loaders.pipelines.xml_common import XML_CLI_SHORTCUTS from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import run_xmltopyarrow_pipeline PIPELINE_NAME: Final[str] = "uniprot_cdm" @@ -27,21 +25,9 @@ UNIREF_LOG_INTERVAL: Final[int] = 10000 -class _CdmXmlPresetMixin: - """Shared class-variable overrides disabling user-supplied transform and output-schema modules.""" - - transforms_module: ClassVar[str | None] = None - output_schemas_module: ClassVar[str | None] = None - - -class UniProtSettings(_CdmXmlPresetMixin, UniProtXmlToPyArrowSettings): +class UniProtSettings(LockedTransformHooksMixin, UniProtXmlToPyArrowSettings, cli_prog_name=PIPELINE_NAME): """Configuration for running the UniProtKB CDM XML-to-PyArrow import pipeline.""" - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=PIPELINE_NAME, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - dataset_name: ClassVar[str] = PIPELINE_NAME log_interval: Annotated[ @@ -62,16 +48,11 @@ def _resolve_settings_schema_map(self, default_pyarrow_schema: pa.Schema | None ) -class UniRefCdmSettings(_CdmXmlPresetMixin, UniRefXmlToPyArrowSettings): +class UniRefCdmSettings(LockedTransformHooksMixin, UniRefXmlToPyArrowSettings, cli_prog_name=UNIREF_CDM_PIPELINE_NAME): """Base settings for UniRef CDM XML-to-PyArrow ingestion pipelines.""" PROTOCOL: ClassVar[str] - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=UNIREF_CDM_PIPELINE_NAME, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - log_interval: Annotated[ LogInterval, Field(default=UNIREF_LOG_INTERVAL), @@ -90,42 +71,24 @@ def _resolve_settings_schema_map(self, default_pyarrow_schema: pa.Schema | None ) -class UniRef50CdmSettings(UniRefCdmSettings): +class UniRef50CdmSettings(UniRefCdmSettings, cli_prog_name=UNIREF50_CDM): """UniRef 50 CDM XML-to-PyArrow ingest settings.""" PROTOCOL: ClassVar[str] = "UniRef 50" - - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=UNIREF50_CDM, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - dataset_name: ClassVar[str] = UNIREF50_CDM -class UniRef90CdmSettings(UniRefCdmSettings): +class UniRef90CdmSettings(UniRefCdmSettings, cli_prog_name=UNIREF90_CDM): """UniRef 90 CDM XML-to-PyArrow ingest settings.""" PROTOCOL: ClassVar[str] = "UniRef 90" - - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=UNIREF90_CDM, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - dataset_name: ClassVar[str] = UNIREF90_CDM -class UniRef100CdmSettings(UniRefCdmSettings): +class UniRef100CdmSettings(UniRefCdmSettings, cli_prog_name=UNIREF100_CDM): """UniRef 100 CDM XML-to-PyArrow ingest settings.""" PROTOCOL: ClassVar[str] = "UniRef 100" - - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=UNIREF100_CDM, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - dataset_name: ClassVar[str] = UNIREF100_CDM diff --git a/src/cdm_data_loaders/pipelines/presets/uniprot_idmapping.py b/src/cdm_data_loaders/pipelines/presets/uniprot_idmapping.py index afdea267..6864217f 100644 --- a/src/cdm_data_loaders/pipelines/presets/uniprot_idmapping.py +++ b/src/cdm_data_loaders/pipelines/presets/uniprot_idmapping.py @@ -17,6 +17,7 @@ ) from cdm_data_loaders.core.transforms.hints import OutputTableHints from cdm_data_loaders.core.transforms.routing import emit_raw_table +from cdm_data_loaders.core.transforms.settings_mixin import LockedTransformHooksMixin from cdm_data_loaders.datasets.uniprot.idmapping import ( UNIPROT_IDMAPPING_OUTPUT_SCHEMAS, transform_uniprot_id_mapping, @@ -31,6 +32,7 @@ RAW_ID_MAPPING_TABLE_NAME, UNIPROT_ID_COL, ) +from cdm_data_loaders.utils.arrow_schema import with_schema_description PIPELINE_NAME: Final[str] = "uniprot_idmapping" DEFAULT_IDMAPPING_FILE_GLOB: Final[str] = "**/*idmapping*.dat*" @@ -45,7 +47,7 @@ ) -class UniProtIdMappingSettings(XsvIngestSettings): +class UniProtIdMappingSettings(LockedTransformHooksMixin, XsvIngestSettings): """Settings for the UniProt ID mapping XSV ingestion pipeline.""" model_config: SettingsConfigDict = default_settings_with_shortcuts( @@ -55,8 +57,6 @@ class UniProtIdMappingSettings(XsvIngestSettings): dataset_name: ClassVar[str] = PIPELINE_NAME table_name: ClassVar[str] = ID_MAPPING_TABLE_NAME - transforms_module: ClassVar[str | None] = None - output_schemas_module: ClassVar[str | None] = None file_glob: Annotated[ FileGlob, @@ -92,11 +92,7 @@ def _resolve_settings_schema_map(self, default_pyarrow_schema: pa.Schema | None } ) super()._resolve_settings_schema_map(default_pyarrow_schema=default_pyarrow_schema) - raw_metadata = { - **(self.pyarrow_schema.metadata or {}), - b"comment": RAW_ID_MAPPING_DESCRIPTION.encode("utf-8"), - } - self._pyarrow_schema = self.pyarrow_schema.with_metadata(raw_metadata) + self._pyarrow_schema = with_schema_description(self.pyarrow_schema, RAW_ID_MAPPING_DESCRIPTION) if self.use_pyarrow: self._schema_map = ( UNIPROT_IDMAPPING_OUTPUT_SCHEMAS diff --git a/src/cdm_data_loaders/pipelines/presets/uniprot_xml.py b/src/cdm_data_loaders/pipelines/presets/uniprot_xml.py index 4aff8a16..f825673a 100644 --- a/src/cdm_data_loaders/pipelines/presets/uniprot_xml.py +++ b/src/cdm_data_loaders/pipelines/presets/uniprot_xml.py @@ -5,11 +5,8 @@ import pyarrow as pa from pydantic import Field, FilePath -from pydantic_settings import SettingsConfigDict from cdm_data_loaders.core.cli import CliRunner -from cdm_data_loaders.core.settings import default_settings_with_shortcuts -from cdm_data_loaders.pipelines.xml_common import XML_CLI_SHORTCUTS from cdm_data_loaders.pipelines.xmltopyarrow.pipeline import run_xmltopyarrow_pipeline from cdm_data_loaders.pipelines.xmltopyarrow.settings import ( SCHEMAS_DIR, @@ -28,16 +25,10 @@ UNIREF_XSD_FILE: Final[Path] = SCHEMAS_DIR / "uniref" / "uniref.xsd" -class UniProtXmlToPyArrowSettings(PresetXmlToPyArrowSettings): +class UniProtXmlToPyArrowSettings(PresetXmlToPyArrowSettings, cli_prog_name=UNIPROT_PIPELINE_NAME): """Settings for the UniProt XML-to-PyArrow ingestion pipeline.""" PRECOMPUTED_SCHEMA: ClassVar[pa.Schema] = UNIPROT_SCHEMA - - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=UNIPROT_PIPELINE_NAME, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - dataset_name: ClassVar[str] = UNIPROT_PIPELINE_NAME xsd_file: Annotated[ @@ -49,16 +40,11 @@ class UniProtXmlToPyArrowSettings(PresetXmlToPyArrowSettings): ] -class UniRefXmlToPyArrowSettings(PresetXmlToPyArrowSettings): +class UniRefXmlToPyArrowSettings(PresetXmlToPyArrowSettings, cli_prog_name=UNIREF_PIPELINE_NAME): """Settings for the UniRef XML-to-PyArrow ingestion pipeline.""" PRECOMPUTED_SCHEMA: ClassVar[pa.Schema] = UNIREF_SCHEMA - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=UNIREF_PIPELINE_NAME, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - xsd_file: Annotated[ FilePath, Field( @@ -68,36 +54,21 @@ class UniRefXmlToPyArrowSettings(PresetXmlToPyArrowSettings): ] -class UniRef50XmlToPyArrowSettings(UniRefXmlToPyArrowSettings): +class UniRef50XmlToPyArrowSettings(UniRefXmlToPyArrowSettings, cli_prog_name=UNIREF_FIFTY): """UniRef 50 XML-to-PyArrow ingest settings.""" - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=UNIREF_FIFTY, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - dataset_name: ClassVar[str] = UNIREF_FIFTY -class UniRef90XmlToPyArrowSettings(UniRefXmlToPyArrowSettings): +class UniRef90XmlToPyArrowSettings(UniRefXmlToPyArrowSettings, cli_prog_name=UNIREF_NINETY): """UniRef 90 XML-to-PyArrow ingest settings.""" - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=UNIREF_NINETY, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - dataset_name: ClassVar[str] = UNIREF_NINETY -class UniRef100XmlToPyArrowSettings(UniRefXmlToPyArrowSettings): +class UniRef100XmlToPyArrowSettings(UniRefXmlToPyArrowSettings, cli_prog_name=UNIREF_HUNDRED): """UniRef 100 XML-to-PyArrow ingest settings.""" - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=UNIREF_HUNDRED, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - dataset_name: ClassVar[str] = UNIREF_HUNDRED diff --git a/src/cdm_data_loaders/pipelines/xmltodict/settings.py b/src/cdm_data_loaders/pipelines/xmltodict/settings.py index be19def0..ad66b2fa 100644 --- a/src/cdm_data_loaders/pipelines/xmltodict/settings.py +++ b/src/cdm_data_loaders/pipelines/xmltodict/settings.py @@ -2,21 +2,10 @@ from typing import Final -from pydantic_settings import SettingsConfigDict - -from cdm_data_loaders.core.settings import default_settings_with_shortcuts -from cdm_data_loaders.pipelines.xml_common import ( - XML_CLI_SHORTCUTS, - SingleTableXmlIngestSettings, -) +from cdm_data_loaders.pipelines.xml_common import SingleTableXmlIngestSettings PIPELINE_NAME: Final[str] = "xmltodict" -class XmlToDictSettings(SingleTableXmlIngestSettings): +class XmlToDictSettings(SingleTableXmlIngestSettings, cli_prog_name=PIPELINE_NAME): """Settings for the XML ingestion pipeline.""" - - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=PIPELINE_NAME, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py index 761987f2..338961cb 100644 --- a/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py @@ -26,12 +26,11 @@ def run_xmltopyarrow_pipeline(settings: XmlToPyArrowSettings) -> LoadInfo | None name="xmltopyarrow_reader", parallelized=False, ) - pipeline_name = str(settings.model_config.get("cli_prog_name") or PIPELINE_NAME) return run_pipeline( settings=settings, resource=xmltopyarrow_resource, - **build_file_ingest_kwargs(settings, pipeline_name), + **build_file_ingest_kwargs(settings, PIPELINE_NAME), ) diff --git a/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py b/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py index 3d2747ef..591857c2 100644 --- a/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py +++ b/src/cdm_data_loaders/pipelines/xmltopyarrow/settings.py @@ -9,7 +9,6 @@ import xmlschema from lxml.etree import Element from pydantic import Field, PositiveInt, PrivateAttr -from pydantic_settings import SettingsConfigDict from cdm_data_loaders.coercion.xml import ( make_xml_element_converter, @@ -30,12 +29,8 @@ RecordTransformFn, TransformRegistry, ) -from cdm_data_loaders.core.settings import ( - default_settings_with_shortcuts, -) from cdm_data_loaders.core.transforms.settings_mixin import TransformSettingsMixin from cdm_data_loaders.pipelines.xml_common import ( - XML_CLI_SHORTCUTS, XmlIngestSettings, XsdFile, ) @@ -66,14 +61,9 @@ def _classify_xmltopyarrow_xsd_error(exc: Exception) -> XmlToPyArrowConfigError: return UnsupportedXsdTargetError(err_msg) -class XmlToPyArrowSettings(XmlIngestSettings, TransformSettingsMixin): +class XmlToPyArrowSettings(XmlIngestSettings, TransformSettingsMixin, cli_prog_name=PIPELINE_NAME): """Settings for the XML to PyArrow ingestion pipeline.""" - model_config: SettingsConfigDict = default_settings_with_shortcuts( - cli_prog_name=PIPELINE_NAME, - cli_shortcuts=XML_CLI_SHORTCUTS, - ) - _xsd_schema: xmlschema.XMLSchema = PrivateAttr() _xml_element_converter: Callable[[Element], dict[str, Any]] | None = PrivateAttr(default=None) _xml_level_converters: dict[str, Callable[[Element], dict[str, Any]]] = PrivateAttr(default_factory=dict) @@ -140,22 +130,7 @@ def _resolve_level_schema_map(self) -> None: self._validate_configured_tables(valid_tables) except ValueError as e: raise InvalidXmlPipelineHookError(str(e)) from e - if not self.use_pyarrow: - self._schema_map = None - return - resolved_schemas: dict[str, pa.Schema] = { - **( - dict(self._unreconciled_output_schemas.registry) - if self._unreconciled_output_schemas is not None - else {} - ), - **{ - table_name: table_schema - for table_name, table_schema in self._pyarrow_schemas.items() - if self._needs_default_pyarrow_schema(table_name) - }, - } - self._schema_map = ArrowSchemaRegistry(registry=resolved_schemas) if resolved_schemas else None + self._resolve_multi_table_schema_map(self._pyarrow_schemas) def _resolve_settings_schema_map(self, default_pyarrow_schema: pa.Schema | None = None) -> None: """Validate target selection mode, load the XSD, and resolve `_schema_map`.""" diff --git a/src/cdm_data_loaders/pipelines/xsv/pipeline.py b/src/cdm_data_loaders/pipelines/xsv/pipeline.py index 5209616c..44d9770d 100644 --- a/src/cdm_data_loaders/pipelines/xsv/pipeline.py +++ b/src/cdm_data_loaders/pipelines/xsv/pipeline.py @@ -230,13 +230,12 @@ def run_xsv_pipeline(settings: XsvIngestSettings) -> LoadInfo | None: :return: load information for the pipeline :rtype: LoadInfo | None """ - pipeline_name = str(settings.model_config.get("cli_prog_name") or PIPELINE_NAME) if settings.loader_file_format == LoaderFileFormatEnum.FILE_LISTING: fs_resource = filesystem(bucket_url=str(settings.input_dir), file_glob=settings.file_glob) return run_pipeline( settings=settings, resource=fs_resource, - **build_file_ingest_kwargs(settings, pipeline_name), + **build_file_ingest_kwargs(settings, PIPELINE_NAME), ) qsv_cmd = qsv_check() @@ -271,7 +270,7 @@ def run_xsv_pipeline(settings: XsvIngestSettings) -> LoadInfo | None: return run_pipeline( settings=settings, resource=xsv_resource, - **build_file_ingest_kwargs(settings, pipeline_name), + **build_file_ingest_kwargs(settings, PIPELINE_NAME), ) diff --git a/src/cdm_data_loaders/utils/arrow_schema.py b/src/cdm_data_loaders/utils/arrow_schema.py index 84fbc8b1..3b2ee535 100644 --- a/src/cdm_data_loaders/utils/arrow_schema.py +++ b/src/cdm_data_loaders/utils/arrow_schema.py @@ -12,10 +12,28 @@ pa.ListViewType, pa.LargeListViewType, ) +SCHEMA_COMMENT_METADATA_KEY: Final[bytes] = b"comment" type ArrowListType = pa.ListType | pa.LargeListType | pa.FixedSizeListType | pa.ListViewType | pa.LargeListViewType +def with_schema_description(schema: pa.Schema, description: str) -> pa.Schema: + """Return a copy of ``schema`` with UTF-8 ``b"comment"`` metadata set to ``description``. + + :param schema: Input PyArrow schema. + :type schema: pa.Schema + :param description: Table description text stored under ``b"comment"``. + :type description: str + :return: PyArrow schema with updated ``b"comment"`` metadata. + :rtype: pa.Schema + """ + merged_metadata = { + **(schema.metadata or {}), + SCHEMA_COMMENT_METADATA_KEY: description.encode("utf-8"), + } + return schema.with_metadata(merged_metadata) + + def assemble_pyarrow_schema( component: pa.StructType | Sequence[pa.Field], *, diff --git a/src/cdm_data_loaders/utils/checksums.py b/src/cdm_data_loaders/utils/checksums.py index 0900d25c..60f2f0de 100644 --- a/src/cdm_data_loaders/utils/checksums.py +++ b/src/cdm_data_loaders/utils/checksums.py @@ -81,15 +81,15 @@ def b64digest(self) -> str: return base64.b64encode(self.digest()).decode() -def _final_digest(hasher: Hasher) -> str: - """Return the canonical digest representation for a completed hasher. +def finalize_digest(hasher: Hasher) -> str: + """Return the canonical digest string for a completed hasher. - Uses the base64 form for crc64nvme (matching S3's ChecksumCRC64NVME - representation) and the hex form for everything else. + Uses the base64 form for ``crc64nvme`` (matching S3's ``ChecksumCRC64NVME`` + representation) and lowercase hex for all other algorithms. - :param hasher: a hasher that has already consumed all its input data + :param hasher: hasher that has consumed all input bytes :type hasher: Hasher - :return: hex digest, or base64 digest for crc64nvme + :return: hex digest, or base64 digest for ``crc64nvme`` :rtype: str """ if isinstance(hasher, Crc64NvmeHasher): @@ -193,7 +193,7 @@ def compute_stream_checksum( hasher: Hasher = new_hasher(algorithm) for chunk in iter(lambda: stream.read(chunk_size), b""): hasher.update(chunk) - return _final_digest(hasher) + return finalize_digest(hasher) def compute_file_checksum(file_path: str | Path, algorithm: str) -> str: diff --git a/tests/cdm_data_loaders/cli/test_ncbi_ftp_filter_updates.py b/tests/cdm_data_loaders/cli/test_ncbi_ftp_filter_updates.py new file mode 100644 index 00000000..e451b562 --- /dev/null +++ b/tests/cdm_data_loaders/cli/test_ncbi_ftp_filter_updates.py @@ -0,0 +1,145 @@ +"""Unit tests for ``cdm_data_loaders.cli.ncbi_ftp_filter_updates``.""" + +from datetime import date +from pathlib import Path + +import pytest +from pydantic import ValidationError +from pydantic_settings import CliApp + +from cdm_data_loaders.cli.ncbi_ftp_filter_updates import ( + FilterNcbiUpdatesSettings, + cli, + run_filter_ncbi_updates, +) +from cdm_data_loaders.core.cli import CliRunner +from cdm_data_loaders.datasets.ncbi_ftp.constants import Source + + +@pytest.fixture +def input_files(tmp_path: Path) -> tuple[Path, Path]: + """Write sample modification-times and assemblies-of-interest files under tmp_path.""" + mod_times = tmp_path / "ftp_mod_times.txt" + mod_times.write_text( + "GCF_000000001.1_RefBefore\t2024/01/10\n" + "GCF_000000002.1_RefAfterUnlisted\t2024/06/15\n" + "GCA_000000003.1_GenAfterListed\t2024/07/01\n" + "GCA_000000004.1_GenAfterUnlisted\t2024/08/01\n", + encoding="utf-8", + ) + interesting = tmp_path / "assemblies.txt" + interesting.write_text("GCF_000000001.1\nGCA_000000003.1\n", encoding="utf-8") + return mod_times, interesting + + +def test_cli_runner_wiring() -> None: + """Verify cli binds FilterNcbiUpdatesSettings to run_filter_ncbi_updates.""" + assert isinstance(cli, CliRunner) + assert cli.settings_cls is FilterNcbiUpdatesSettings + assert cli.pipeline_fn is run_filter_ncbi_updates + + +@pytest.mark.parametrize( + ("raw_date", "expected_date"), + [ + pytest.param("2024/05/01", date(2024, 5, 1), id="slash-separated"), + pytest.param("2024-05-01", date(2024, 5, 1), id="iso-hyphenated"), + pytest.param(date(2024, 5, 1), date(2024, 5, 1), id="date-instance"), + ], +) +def test_settings_date_formats_and_paths( + input_files: tuple[Path, Path], + tmp_path: Path, + raw_date: date | str, + expected_date: date, +) -> None: + """Accept YYYY/MM/DD, YYYY-MM-DD, and date objects and expose derived directories.""" + mod_times, interesting = input_files + out_dir = tmp_path / "out" + settings = FilterNcbiUpdatesSettings( + ftp_mod_times_path=mod_times, + assemblies_of_interest_path=interesting, + mod_date=raw_date, # type: ignore[arg-type] + output_dir=out_dir, + ) + assert settings.mod_date == expected_date + assert settings.mod_date_tag == "20240501" + assert settings.derived_dir == out_dir / "derived" + assert settings.source_dir(Source.REFSEQ) == out_dir / "refseq" + assert settings.source_dir(Source.GENBANK) == out_dir / "genbank" + + +@pytest.mark.parametrize( + ("field_overrides", "match"), + [ + pytest.param({"mod_date": "not-a-date"}, "Invalid isoformat string", id="invalid-date"), + pytest.param({"batch_size": 0}, "greater than 0", id="zero-batch-size"), + ], +) +def test_settings_validation_errors( + input_files: tuple[Path, Path], + tmp_path: Path, + field_overrides: dict[str, object], + match: str, +) -> None: + """Reject malformed modification dates and non-positive batch sizes.""" + mod_times, interesting = input_files + kwargs: dict[str, object] = { + "ftp_mod_times_path": mod_times, + "assemblies_of_interest_path": interesting, + "mod_date": "2024/05/01", + "output_dir": tmp_path / "out", + **field_overrides, + } + with pytest.raises(ValidationError, match=match): + FilterNcbiUpdatesSettings(**kwargs) # type: ignore[arg-type] + + +@pytest.mark.parametrize( + ("filter_refseq", "expected_refseq_after"), + [ + pytest.param(False, ["GCF_000000002.1"], id="default-retains-all-refseq"), + pytest.param(True, [], id="flag-filters-refseq"), + ], +) +def test_run_filter_ncbi_updates_writes_tables_and_batches( + input_files: tuple[Path, Path], + tmp_path: Path, + filter_refseq: bool, + expected_refseq_after: list[str], +) -> None: + """Write derived TSV tables and batched ID files for each (Source, Period) split.""" + mod_times, interesting = input_files + out_dir = tmp_path / "output" + cli_args = [ + "--ftp-mod-times-path", + str(mod_times), + "--assemblies-of-interest-path", + str(interesting), + "--mod-date", + "2024/05/01", + "--output-dir", + str(out_dir), + "--batch-size", + "1", + ] + if filter_refseq: + cli_args.append("--filter-refseq") + + settings = CliApp.run(FilterNcbiUpdatesSettings, cli_args=cli_args) + run_filter_ncbi_updates(settings) + + assert (out_dir / "derived" / "ftp_mod_times_parsed.tsv").is_file() + assert (out_dir / "derived" / "interesting_assemblies.tsv").is_file() + assert (out_dir / "derived" / "refseq_before_20240501.tsv").is_file() + assert (out_dir / "derived" / "genbank_after_20240501.tsv").is_file() + + assert (out_dir / "refseq" / "before_20240501-1.txt").read_text(encoding="utf-8") == "GCF_000000001.1\n" + assert (out_dir / "genbank" / "after_20240501-1.txt").read_text(encoding="utf-8") == "GCA_000000003.1\n" + assert not (out_dir / "genbank" / "before_20240501-1.txt").exists() + + refseq_after_batch = out_dir / "refseq" / "after_20240501-1.txt" + if expected_refseq_after: + assert refseq_after_batch.read_text(encoding="utf-8").splitlines() == expected_refseq_after + else: + assert not refseq_after_batch.exists() diff --git a/tests/cdm_data_loaders/core/test_buffer.py b/tests/cdm_data_loaders/core/test_buffer.py index 9d426fd3..74120e9a 100644 --- a/tests/cdm_data_loaders/core/test_buffer.py +++ b/tests/cdm_data_loaders/core/test_buffer.py @@ -13,6 +13,9 @@ DEFAULT_BUFFER_MAX_ITEMS, REJECTED_PYARROW_SCHEMA, DictBuffer, + format_rejected_records, + make_rejected_row, + resolve_destination_schema, ) from cdm_data_loaders.core.fields import REJECTED_TABLE_NAME from cdm_data_loaders.core.registry import ArrowSchemaRegistry @@ -423,3 +426,39 @@ def test_dict_buffer_pass_loses_nothing_under_parallelized_transformer() -> None values = _run_page_per_call_transformer(total=total, max_items=5, chunk_size=7) assert sorted(values) == list(range(total)) + + +@pytest.mark.parametrize( + ("table_name", "schema_map", "expected_schema"), + [ + pytest.param(REJECTED_TABLE_NAME, None, REJECTED_PYARROW_SCHEMA, id="rejected-without-schema-map"), + pytest.param(REJECTED_TABLE_NAME, PEOPLE_SCHEMA_MAP, REJECTED_PYARROW_SCHEMA, id="rejected-with-schema-map"), + pytest.param(PEOPLE, PEOPLE_SCHEMA_MAP, PEOPLE_SCHEMA, id="mapped-table"), + pytest.param("unmapped", PEOPLE_SCHEMA_MAP, None, id="unmapped-table"), + pytest.param(PEOPLE, None, None, id="none-schema-map"), + ], +) +def test_resolve_destination_schema_resolves_rejected_and_mapped_tables( + table_name: str, + schema_map: ArrowSchemaRegistry | None, + expected_schema: pa.Schema | None, +) -> None: + """resolve_destination_schema returns REJECTED_PYARROW_SCHEMA for _rejected and looks up other tables.""" + assert resolve_destination_schema(table_name, schema_map) == expected_schema + + +@pytest.mark.parametrize("use_pyarrow", [True, False], ids=["pyarrow-page", "dict-list-page"]) +def test_format_rejected_records_formats_pyarrow_and_dict_pages(use_pyarrow: bool) -> None: + """format_rejected_records wraps rejected rows with _rejected table metadata as PyArrow or list pages.""" + rows = [ + make_rejected_row("people", "bad row", source_file="people.csv", line_no=2, raw_record="x"), + ] + item = format_rejected_records(rows, use_pyarrow=use_pyarrow) + + assert item.meta.table_name == REJECTED_TABLE_NAME + if use_pyarrow: + assert isinstance(item.data, pa.Table) + assert item.data.schema.equals(REJECTED_PYARROW_SCHEMA) + assert item.data.to_pylist() == rows + else: + assert item.data == rows diff --git a/tests/cdm_data_loaders/core/test_filesystem.py b/tests/cdm_data_loaders/core/test_filesystem.py index 86094f6d..ede9c14f 100644 --- a/tests/cdm_data_loaders/core/test_filesystem.py +++ b/tests/cdm_data_loaders/core/test_filesystem.py @@ -29,6 +29,7 @@ find_filesystem_resources, list_input_dir_files, make_per_file_reader, + resolve_pipeline_name, ) from cdm_data_loaders.core.preprocessors import FilePreprocessError from cdm_data_loaders.core.settings import ( @@ -794,3 +795,45 @@ def test_build_file_ingest_kwargs_pass( }, "pipeline_run_kwargs": expected_run_kwargs, } + + +@pytest.mark.parametrize( + ("settings_obj", "expected"), + [ + pytest.param(type("Cfg", (), {"model_config": {"cli_prog_name": "preset_cli"}})(), "preset_cli", id="prog-set"), + pytest.param(type("Cfg", (), {"model_config": {"cli_prog_name": None}})(), "default_pipe", id="prog-none"), + pytest.param(type("Cfg", (), {"model_config": {"cli_prog_name": ""}})(), "default_pipe", id="prog-empty"), + pytest.param(object(), "default_pipe", id="no-model-config"), + ], +) +def test_resolve_pipeline_name_prefers_cli_prog_name_or_falls_back( + settings_obj: object, + expected: str, +) -> None: + """resolve_pipeline_name returns cli_prog_name when non-empty and falls back to default_pipeline_name.""" + assert resolve_pipeline_name(settings_obj, "default_pipe") == expected + + +def test_build_file_ingest_kwargs_prefers_subclass_cli_prog_name(tmp_path: Path) -> None: + """build_file_ingest_kwargs uses settings.model_config['cli_prog_name'] for pipeline_name and dataset fallback.""" + + class PresetIngestSettings(FileFormatIngestSettings, cli_prog_name="preset_ingest"): + pass + + base_settings: PresetIngestSettings = make_cts_settings( + PresetIngestSettings, + tmp_path, + dataset_name="unused_ds", + loader_file_format="parquet", + ) + settings = base_settings.model_copy(update={"dataset_name": None}) + + result = build_file_ingest_kwargs(settings, TINY_PIPELINE_NAME, fallback_dataset_name=True) + + assert result == { + "pipeline_kwargs": { + "pipeline_name": "preset_ingest", + "dataset_name": "preset_ingest", + }, + "pipeline_run_kwargs": {"loader_file_format": "parquet"}, + } diff --git a/tests/cdm_data_loaders/core/test_transforms.py b/tests/cdm_data_loaders/core/test_transforms.py index 0c99a54e..0ae15ad5 100644 --- a/tests/cdm_data_loaders/core/test_transforms.py +++ b/tests/cdm_data_loaders/core/test_transforms.py @@ -16,6 +16,7 @@ ) from frozendict import frozendict from pydantic import BaseModel, SecretStr, ValidationError +from pydantic_settings import CliApp, SettingsError from cdm_data_loaders.coercion.pydantic_to_arrow import coerce_pydantic_record_for_pyarrow from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA, make_rejected_row @@ -45,12 +46,16 @@ route_and_coerce_table, validate_routed_output, ) -from cdm_data_loaders.core.transforms.settings_mixin import validate_table_transforms +from cdm_data_loaders.core.transforms.settings_mixin import ( + LockedTransformHooksMixin, + reconcile_multi_table_schema_map, + validate_table_transforms, +) from cdm_data_loaders.pipelines.jsonlines.routing import route_jsonl_lines from cdm_data_loaders.pipelines.jsonlines.settings import JsonlExtractSettings from cdm_data_loaders.pipelines.xmltodict.processors import process_xml_file_to_dict from cdm_data_loaders.pipelines.xmltodict.settings import XmlToDictSettings -from tests.helpers.cli_assertions import make_cts_settings, register_temp_module +from tests.helpers.cli_assertions import make_cts_settings, register_temp_module, settings_to_argv def _identity_items_transform(record: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: @@ -1264,3 +1269,109 @@ class SimpleItem(BaseModel): router = make_record_router("items", transform=_identity_items_transform) with pytest.raises(TypeError, match="TableTransformFn requires a dict"): router(SimpleItem(id="a"), None) + + +def test_make_record_router_coerces_rejected_rows_when_schema_map_omits_rejected() -> None: + """make_record_router coerces _rejected rows against REJECTED_PYARROW_SCHEMA when schema_map omits _rejected.""" + + def _reject_item(row: dict[str, Any], ctx: TransformContext) -> dict[str, list[dict[str, Any]]]: + return { + "_rejected": [ + { + "table_name": ctx.table_name, + "source_file": ctx.source_file, + "line_no": "7", + "raw_record": str(row.get("id")), + "error_detail": "invalid item", + "undeclared_extra": "dropped", + } + ] + } + + router = make_record_router("items", transform=_reject_item, schema_map=ITEMS_SCHEMA_REGISTRY) + assert router({"id": 42}, "batch/items.jsonl") == { + "_rejected": [ + { + "table_name": "items", + "source_file": "batch/items.jsonl", + "line_no": 7, + "raw_record": "42", + "error_detail": "invalid item", + } + ] + } + + +def test_locked_transform_hooks_mixin_locks_transforms_and_output_schemas_off_cli( + test_defaults: dict[str, Any], +) -> None: + """LockedTransformHooksMixin removes transforms_module and output_schemas_module from CLI while keeping preprocessors_module.""" + + class LockedExtractSettings(LockedTransformHooksMixin, JsonlExtractSettings, cli_prog_name="locked_extract"): + pass + + assert "transforms_module" not in LockedExtractSettings.model_fields + assert "output_schemas_module" not in LockedExtractSettings.model_fields + assert "preprocessors_module" in LockedExtractSettings.model_fields + + settings = LockedExtractSettings(**test_defaults, dataset_name="ds", table_name="items") + assert settings.transforms_module is None + assert settings.output_schemas_module is None + + argv = [*settings_to_argv(settings), "--transforms-module", "some.mod"] + with pytest.raises(SettingsError, match="--transforms-module"): + CliApp.run(LockedExtractSettings, cli_args=argv) + + +@pytest.mark.parametrize( + ("use_pyarrow", "transformed_tables", "explicit_schemas", "expected_built", "expected_tables"), + [ + pytest.param(False, (), {}, [], None, id="use-pyarrow-false"), + pytest.param(True, (), {}, ["alpha", "beta"], {"alpha", "beta"}, id="all-defaults"), + pytest.param(True, ("alpha",), {}, ["beta"], {"beta"}, id="skips-transformed-without-explicit-schema"), + pytest.param( + True, + ("alpha",), + {"alpha_out": SOURCED_ITEMS_SCHEMA}, + ["beta"], + {"alpha_out", "beta"}, + id="merges-explicit-fanout-and-untransformed-default", + ), + pytest.param(True, ("alpha", "beta"), {}, [], None, id="all-transformed-unschematized"), + ], +) +def test_reconcile_multi_table_schema_map_reconciles_defaults_and_overrides( + use_pyarrow: bool, + transformed_tables: tuple[str, ...], + explicit_schemas: dict[str, pa.Schema], + expected_built: list[str], + expected_tables: set[str] | None, +) -> None: + """reconcile_multi_table_schema_map builds defaults only for untransformed unmapped tables and merges explicit schemas.""" + built: list[str] = [] + + def _build_default(table_name: str) -> pa.Schema | None: + built.append(table_name) + return ITEMS_SCHEMA + + transforms = ( + TransformRegistry(registry=dict.fromkeys(transformed_tables, _identity_items_transform)) + if transformed_tables + else None + ) + out_schemas = ArrowSchemaRegistry(registry=explicit_schemas) if explicit_schemas else None + + result = reconcile_multi_table_schema_map( + ["alpha", "beta"], + _build_default, + use_pyarrow=use_pyarrow, + table_transforms=transforms, + output_schemas=out_schemas, + ) + + assert built == expected_built + if expected_tables is None: + assert result is None + else: + assert result is not None + assert set(result.registry) == expected_tables diff --git a/tests/cdm_data_loaders/datasets/all_the_bacteria/test_osf.py b/tests/cdm_data_loaders/datasets/all_the_bacteria/test_osf.py index d6ccfa24..bab95c80 100644 --- a/tests/cdm_data_loaders/datasets/all_the_bacteria/test_osf.py +++ b/tests/cdm_data_loaders/datasets/all_the_bacteria/test_osf.py @@ -1,11 +1,19 @@ -"""Unit tests for ``load_patterns`` in ``cdm_data_loaders.datasets.all_the_bacteria.osf``.""" +"""Unit tests for ``cdm_data_loaders.datasets.all_the_bacteria.osf``.""" +import logging import re from pathlib import Path +from unittest.mock import MagicMock, patch import pytest -from cdm_data_loaders.datasets.all_the_bacteria.osf import load_patterns +from cdm_data_loaders.datasets.all_the_bacteria.osf import ( + ALL_ATB_FILE_NAME, + download_osf_files, + download_osf_index_tsv, + iter_atb_download_links, + load_patterns, +) @pytest.fixture @@ -107,3 +115,108 @@ def test_load_patterns_empty_file_returns_none(tmp_path: Path) -> None: path = tmp_path / "patterns.txt" path.write_text("\n\n \t\n \n \n\t\t\n", encoding="utf-8") assert load_patterns(path) is None + + +@patch("cdm_data_loaders.datasets.all_the_bacteria.osf.FileDownloader") +@patch("cdm_data_loaders.datasets.all_the_bacteria.osf.fetch_json") +def test_download_osf_index_tsv_fetches_link_and_downloads( + mock_fetch_json: MagicMock, + mock_downloader_cls: MagicMock, + tmp_path: Path, +) -> None: + """Fetch OSF metadata JSON and download all_atb_files.tsv into raw_data_dir.""" + mock_fetch_json.return_value = {"data": {"links": {"download": "https://osf.io/download/r6gcp/"}}} + raw_dir = tmp_path / "raw" + result = download_osf_index_tsv(raw_dir) + + expected_dest = raw_dir / ALL_ATB_FILE_NAME + assert result == expected_dest + mock_downloader_cls.return_value.download.assert_called_once_with( + url="https://osf.io/download/r6gcp/", + destination=expected_dest, + ) + + +def test_download_osf_index_tsv_missing_link_raises(tmp_path: Path) -> None: + """Raise RuntimeError when the OSF metadata JSON lacks data.links.download.""" + with ( + patch("cdm_data_loaders.datasets.all_the_bacteria.osf.fetch_json", return_value={"data": {"links": {}}}), + pytest.raises(RuntimeError, match="Could not find download URL"), + ): + download_osf_index_tsv(tmp_path / "raw") + + +def test_iter_atb_download_links_filters_and_warns_on_extra_column( + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Yield matching rows and log WARNING when non-required headers change.""" + tsv = tmp_path / "index.tsv" + tsv.write_text( + "project\tfilename\turl\tmd5\textra_col\n" + "AllTheBacteria/Metadata\tspecies.tsv.gz\thttps://osf.io/1\taaa\tx\n" + "AllTheBacteria/Other\tskip.tsv.gz\thttps://osf.io/2\tbbb\ty\n", + encoding="utf-8", + ) + with caplog.at_level(logging.WARNING): + batches = list(iter_atb_download_links(tsv, re.compile(r"^AllTheBacteria/Metadata$"))) + + assert batches == [ + [ + { + "project": "AllTheBacteria/Metadata", + "filename": "species.tsv.gz", + "url": "https://osf.io/1", + "md5": "aaa", + "extra_col": "x", + } + ] + ] + assert "ATB file index TSV headers have changed" in caplog.text + + +@pytest.mark.parametrize( + ("tsv_text", "match"), + [ + pytest.param("project\tproject_id\tfilename\turl\tmd5\tsize(MB)\n", "No valid TSV data found", id="empty-rows"), + pytest.param("project\tfilename\turl\nA\tf.tsv\thttps://osf.io/1\n", "Missing required", id="missing-md5-col"), + ], +) +def test_iter_atb_download_links_invalid_tsv(tmp_path: Path, tsv_text: str, match: str) -> None: + """Raise RuntimeError on empty TSV bodies or missing required headers.""" + tsv = tmp_path / "bad.tsv" + tsv.write_text(tsv_text, encoding="utf-8") + with pytest.raises(RuntimeError, match=match): + list(iter_atb_download_links(tsv, re.compile(r".*"))) + + +@patch("cdm_data_loaders.datasets.all_the_bacteria.osf.FileDownloader") +def test_download_osf_files_records_successes_and_skips_failures( + mock_downloader_cls: MagicMock, + tmp_path: Path, +) -> None: + """Populate 'path' on downloaded file records and continue past single-file download errors.""" + mock_client = mock_downloader_cls.return_value + + def side_effect(*, url: str, destination: Path, expected_checksum: str, checksum_fn: str) -> Path: + assert checksum_fn == "md5" + if url.endswith("/bad"): + raise OSError + assert expected_checksum == "abc" + return destination + + mock_client.download.side_effect = side_effect + files = [ + {"project": "AllTheBacteria/Metadata/", "filename": "ok.tsv.gz", "url": "https://osf.io/ok", "md5": "abc"}, + {"project": "AllTheBacteria/Assembly/", "filename": "bad.tar.xz", "url": "https://osf.io/bad", "md5": "def"}, + ] + succeeded = download_osf_files(tmp_path, files) + assert succeeded == [ + { + "project": "AllTheBacteria/Metadata/", + "filename": "ok.tsv.gz", + "url": "https://osf.io/ok", + "md5": "abc", + "path": str(tmp_path / "Metadata" / "ok.tsv.gz"), + } + ] diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py index 94f3109e..17d2da9c 100644 --- a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py +++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_assembly.py @@ -1,14 +1,22 @@ -"""Tests for ncbi_ftp.assembly module — path helpers, file filtering, checksum parsing.""" +"""Tests for ncbi_ftp.assembly module — path helpers, file filtering, and FTP assembly downloads.""" -from pathlib import PurePosixPath +from collections.abc import Callable +from ftplib import FTP +from pathlib import Path, PurePosixPath +from unittest.mock import MagicMock import pytest from cdm_data_loaders.datasets.ncbi_ftp.assembly import ( + MAX_DOWNLOAD_ATTEMPTS, build_accession_path, + download_assembly_to_local, + extract_accession, + filter_assembly_checksums, + matches_assembly_file_filter, parse_assembly_path, ) -from cdm_data_loaders.utils.checksums import parse_md5_checksums_file +from cdm_data_loaders.utils.checksums import new_hasher, parse_md5_checksums_file from tests.cdm_data_loaders.datasets.ncbi_ftp._helpers import ACC_PATH_215 # Path helpers @@ -86,3 +94,160 @@ def test_parse_assembly_path_invalid() -> None: def test_parse_md5_checksums_file(text: str, expected: dict[PurePosixPath, str]) -> None: """Verify parse_md5_checksums_file handles various input formats.""" assert parse_md5_checksums_file(text) == expected + + +@pytest.mark.parametrize( + ("entry", "expected"), + [ + pytest.param( + "/genomes/all/GCF/000/001/215/GCF_000001215.4_Release_6_plus_ISO1_MT/", + "GCF_000001215.4", + id="ftp-dir-path", + ), + pytest.param( + PurePosixPath("cdm-lake/ncbi-refseq/raw_data/GCA/012/345/678/GCA_012345678.10_ASM123v10/file.fna.gz"), + "GCA_012345678.10", + id="s3-object-key", + ), + pytest.param("GCF_000005845.2_ASM584v2", "GCF_000005845.2", id="assembly-dir-name"), + pytest.param(" GCF_000005845.20 ", "GCF_000005845.20", id="bare-accession-padded"), + ], +) +def test_extract_accession_valid(entry: str | PurePosixPath, expected: str) -> None: + """Extract exact versioned accessions from FTP paths, S3 keys, directory names, and bare IDs.""" + assert extract_accession(entry) == expected + + +@pytest.mark.parametrize( + "entry", + [ + pytest.param("", id="empty"), + pytest.param(" ", id="whitespace"), + pytest.param("GCF_000005845.2extra", id="trailing-alphanumeric-without-underscore"), + pytest.param("XYZ_000005845.2", id="unknown-prefix"), + pytest.param("/genomes/all/random/folder", id="unparseable-path"), + ], +) +def test_extract_accession_invalid(entry: str) -> None: + """Reject empty, malformed, or non-RefSeq/GenBank entries with ValueError.""" + with pytest.raises(ValueError, match="Cannot parse accession"): + extract_accession(entry) + + +@pytest.mark.parametrize( + ("filename", "expected"), + [ + pytest.param(PurePosixPath("GCF_000001215.4_genomic.fna.gz"), True, id="genomic-fna"), + pytest.param(PurePosixPath("sub/GCF_000001215.4_protein.faa.gz"), True, id="nested-protein-faa"), + pytest.param(PurePosixPath("GCF_000001215.4_assembly_report.txt"), True, id="assembly-report"), + pytest.param(PurePosixPath("md5checksums.txt"), False, id="md5checksums"), + pytest.param(PurePosixPath("README.txt"), False, id="readme"), + ], +) +def test_matches_assembly_file_filter(filename: PurePosixPath, expected: bool) -> None: + """Match target assembly file suffixes and exclude unrelated files.""" + assert matches_assembly_file_filter(filename) is expected + + +def test_filter_assembly_checksums() -> None: + """Retain checksum entries matching FILE_FILTERS and drop non-matching files.""" + raw = { + PurePosixPath("GCF_000001215.4_genomic.fna.gz"): "aaa", + PurePosixPath("GCF_000001215.4_assembly_stats.txt"): "bbb", + PurePosixPath("annotation_hashes.txt"): "ccc", + } + assert filter_assembly_checksums(raw) == { + PurePosixPath("GCF_000001215.4_genomic.fna.gz"): "aaa", + PurePosixPath("GCF_000001215.4_assembly_stats.txt"): "bbb", + } + + +def _md5_hex(data: bytes) -> str: + """Compute lowercase MD5 hex digest for test payloads.""" + hasher = new_hasher("md5") + hasher.update(data) + return hasher.hexdigest() + + +def test_download_assembly_to_local_verifies_md5_and_handles_unlisted_file(tmp_path: Path) -> None: + """Download filtered files, write .md5 sidecar for verified files, and track files lacking checksums.""" + fna_bytes = b">seq1\nACGT\n" + fna_md5 = _md5_hex(fna_bytes) + stats_bytes = b"# stats\n" + md5_manifest = f"{fna_md5} ./GCF_000001215.4_genomic.fna.gz\n" + + mock_ftp = MagicMock(spec=FTP) + + def fake_retrlines(cmd: str, callback: Callable[[str], None]) -> None: + if cmd == "NLST": + for name in [ + "md5checksums.txt", + "GCF_000001215.4_genomic.fna.gz", + "GCF_000001215.4_assembly_stats.txt", + "README.txt", + ]: + callback(name) + elif cmd == "RETR md5checksums.txt": + for line in md5_manifest.splitlines(): + callback(line) + + def fake_retrbinary(cmd: str, callback: Callable[[bytes], None]) -> None: + if cmd.endswith("_genomic.fna.gz"): + callback(fna_bytes) + elif cmd.endswith("_assembly_stats.txt"): + callback(stats_bytes) + + mock_ftp.retrlines.side_effect = fake_retrlines + mock_ftp.retrbinary.side_effect = fake_retrbinary + + assembly_ftp_path = PurePosixPath("/") / "genomes" / "all" / ACC_PATH_215 + result = download_assembly_to_local(assembly_ftp_path, tmp_path, ftp=mock_ftp) + + dest_dir = tmp_path / "raw_data" / ACC_PATH_215 + assert result == { + "accession": "GCF_000001215.4", + "assembly_dir": "GCF_000001215.4_Release_6_plus_ISO1_MT", + "files_downloaded": 3, + "files_skipped_checksum_mismatch": 0, + "files_without_checksum": 1, + } + assert (dest_dir / "md5checksums.txt").read_text(encoding="utf-8") == md5_manifest.rstrip("\n") + assert (dest_dir / "GCF_000001215.4_genomic.fna.gz").read_bytes() == fna_bytes + assert (dest_dir / "GCF_000001215.4_genomic.fna.gz.md5").read_text(encoding="utf-8") == fna_md5 + assert (dest_dir / "GCF_000001215.4_assembly_stats.txt").read_bytes() == stats_bytes + assert not (dest_dir / "GCF_000001215.4_assembly_stats.txt.md5").exists() + assert not (dest_dir / "README.txt").exists() + mock_ftp.quit.assert_not_called() + + +def test_download_assembly_to_local_skips_and_unlinks_after_checksum_mismatch(tmp_path: Path) -> None: + """Retry mismatched MD5 downloads MAX_DOWNLOAD_ATTEMPTS times, then unlink the corrupt file.""" + expected_md5 = _md5_hex(b"expected") + md5_manifest = f"{expected_md5} ./GCF_000001215.4_genomic.fna.gz" + mock_ftp = MagicMock(spec=FTP) + retr_attempts = 0 + + def fake_retrlines(cmd: str, callback: Callable[[str], None]) -> None: + if cmd == "NLST": + callback("md5checksums.txt") + callback("GCF_000001215.4_genomic.fna.gz") + elif cmd == "RETR md5checksums.txt": + callback(md5_manifest) + + def fake_retrbinary(_cmd: str, callback: Callable[[bytes], None]) -> None: + nonlocal retr_attempts + retr_attempts += 1 + callback(b"corrupted-payload") + + mock_ftp.retrlines.side_effect = fake_retrlines + mock_ftp.retrbinary.side_effect = fake_retrbinary + + assembly_ftp_path = PurePosixPath("/") / "genomes" / "all" / ACC_PATH_215 + result = download_assembly_to_local(assembly_ftp_path, tmp_path, ftp=mock_ftp) + + dest_dir = tmp_path / "raw_data" / ACC_PATH_215 + assert retr_attempts == MAX_DOWNLOAD_ATTEMPTS + assert result["files_downloaded"] == 1 + assert result["files_skipped_checksum_mismatch"] == 1 + assert not (dest_dir / "GCF_000001215.4_genomic.fna.gz").exists() + assert not (dest_dir / "GCF_000001215.4_genomic.fna.gz.md5").exists() diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py index 2dce3ab4..eae6041e 100644 --- a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py +++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_manifest.py @@ -9,9 +9,11 @@ import pytest -from cdm_data_loaders.datasets.ncbi_ftp.constants import DEFAULT_DESTINATION_PREFIX +from cdm_data_loaders.datasets.ncbi_ftp.constants import DEFAULT_DESTINATION_PREFIX, Source from cdm_data_loaders.datasets.ncbi_ftp.manifest import ( _extract_accession_dir_and_id_from_s3_key, + filter_unpromoted_manifest_lines, + read_manifest_lines, scan_store_to_synthetic_summary, verify_transfer_candidates, write_diff_summary, @@ -20,10 +22,12 @@ write_updated_manifest, ) from cdm_data_loaders.datasets.ncbi_ftp.summary import ( + SUMMARY_FTP_PATHS, AssemblyRecord, DiffResult, accession_prefix, compute_diff, + download_assembly_summary, filter_by_prefix_range, ftp_dir_from_url, get_latest_assembly_paths, @@ -481,11 +485,19 @@ def test_verify_transfer_candidates_skips_ftp_when_folder_missing( ("GCA_999999999.1_assembly_name", "GCA_999999999.1"), id="subdir", ), + pytest.param( + "cdm-lake/ncbi-refseq/raw_data/GCF/000/005/845/GCF_000005845.20_ASM584v20/file.gz", + ("GCF_000005845.20_ASM584v20", "GCF_000005845.20"), + id="str_key", + ), pytest.param(PurePosixPath("some") / "random" / "path", None, id="no_accession"), pytest.param(PurePosixPath(""), None, id="empty"), ], ) -def test_extract_accession_dir_and_id_from_s3_key(key: PurePosixPath, expected: tuple[str, str] | None) -> None: +def test_extract_accession_dir_and_id_from_s3_key( + key: str | PurePosixPath, + expected: tuple[str, str] | None, +) -> None: """Accession is extracted from S3 key paths; invalid/empty paths return None.""" if expected is None: with pytest.raises(ValueError, match="Could not parse S3 key for accession info"): @@ -637,3 +649,86 @@ def test_scan_store_assembly_dir_survives_round_trip(mock_list_objects: MagicMoc assert reparsed_rec.assembly_dir == original_rec.assembly_dir assert reparsed_rec.seq_rel_date == original_rec.seq_rel_date assert reparsed_rec.status == original_rec.status + + +@pytest.mark.parametrize("use_path", [True, False], ids=["from-path", "from-iterable"]) +def test_read_manifest_lines_strips_blanks_and_comments(tmp_path: Path, use_path: bool) -> None: + """Skip blank lines and '#' comments when reading from a Path or iterable of lines.""" + raw = "# header comment\n\n GCF_000001215.4 \n # indented comment\nGCA_012345678.1\n\n" + if use_path: + manifest_path = tmp_path / "manifest.txt" + manifest_path.write_text(raw, encoding="utf-8") + result = read_manifest_lines(manifest_path) + else: + result = read_manifest_lines(raw.splitlines(keepends=True)) + assert result == ["GCF_000001215.4", "GCA_012345678.1"] + + +def test_filter_unpromoted_manifest_lines_exact_version_and_unparseable() -> None: + """Remove promoted accessions without version-prefix collisions and preserve unparseable entries.""" + text = ( + "# comment\n" + "/genomes/all/GCF/000/005/845/GCF_000005845.1_ASM584v1/\n" + "/genomes/all/GCF/000/005/845/GCF_000005845.10_ASM584v10/\n" + "unparseable_entry\n" + ) + remaining, total = filter_unpromoted_manifest_lines(text, {"GCF_000005845.1"}) + expected_total = 3 + assert total == expected_total + assert remaining == [ + "/genomes/all/GCF/000/005/845/GCF_000005845.10_ASM584v10/", + "unparseable_entry", + ] + + +@patch("cdm_data_loaders.datasets.ncbi_ftp.manifest.list_objects") +def test_scan_store_filters_by_database_and_rejects_unknown_database(mock_list_objects: MagicMock) -> None: + """Scan store filters by GCA_ prefix for genbank and raises ValueError on unknown database.""" + mock_list_objects.return_value = [ + {"Key": "prefix/GCF_000001215.4_Release_6/file.gz", "LastModified": datetime(2024, 1, 1, tzinfo=UTC)}, + {"Key": "prefix/GCA_012345678.1_ASM123v1/file.gz", "LastModified": datetime(2024, 1, 2, tzinfo=UTC)}, + ] + genbank_records = scan_store_to_synthetic_summary( + PurePosixPath("test-bucket"), + PurePosixPath("prefix"), + "2024/01/31", + database=Source.GENBANK, + ) + assert list(genbank_records) == ["GCA_012345678.1"] + + with pytest.raises(ValueError, match="Unknown database"): + scan_store_to_synthetic_summary( + PurePosixPath("test-bucket"), + PurePosixPath("prefix"), + "2024/01/31", + database="invalid_db", + ) + + +@pytest.mark.parametrize( + "database", + [ + pytest.param(Source.REFSEQ, id="refseq"), + pytest.param(Source.GENBANK, id="genbank"), + ], +) +@patch("cdm_data_loaders.datasets.ncbi_ftp.summary.ftp_retrieve_text", return_value="# summary\n") +@patch("cdm_data_loaders.datasets.ncbi_ftp.summary.ftp_session") +def test_download_assembly_summary_supported_databases( + mock_session: MagicMock, + mock_retrieve: MagicMock, + database: Source, +) -> None: + """Fetch the expected FTP summary file path for refseq and genbank.""" + mock_ftp = MagicMock() + mock_session.return_value.__enter__.return_value = mock_ftp + content = download_assembly_summary(database=database, ftp_host="ftp.example.com") + assert content == "# summary\n" + mock_session.assert_called_once_with("ftp.example.com") + mock_retrieve.assert_called_once_with(mock_ftp, SUMMARY_FTP_PATHS[database]) + + +def test_download_assembly_summary_unknown_database() -> None: + """Raise ValueError when database is neither refseq nor genbank.""" + with pytest.raises(ValueError, match="Unknown database: bogus"): + download_assembly_summary(database="bogus") diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_promote.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_promote.py index 43421467..22f6ae91 100644 --- a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_promote.py +++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_promote.py @@ -120,6 +120,15 @@ def test_promote_with_metadata(mock_s3_client: botocore.client.BaseClient) -> No ["GCF_000001215.4"], id="all", ), + pytest.param( + "# header comment\n\n" + "/genomes/all/GCF/000/001/215/GCF_000001215.1_ASM1/\n" + "/genomes/all/GCF/000/001/215/GCF_000001215.10_ASM10/\n", + {"GCF_000001215.1"}, + ["GCF_000001215.10_ASM10"], + ["GCF_000001215.1_ASM1", "# header comment"], + id="version-prefix-no-collision", + ), ], ) def test_trim_manifest( @@ -129,7 +138,7 @@ def test_trim_manifest( expected_present: list[str], expected_absent: list[str], ) -> None: - """Promoted accessions are removed; others remain (partial) or the manifest empties (all).""" + """Promoted accessions are removed; others remain without version-prefix collisions.""" manifest_key = PurePosixPath("manifests/transfer_manifest.txt") mock_s3_client.put_object(Bucket=str(TEST_BUCKET), Key=str(manifest_key), Body=manifest_body.encode()) _trim_manifest(manifest_key, TEST_BUCKET, promoted_set) @@ -140,6 +149,18 @@ def test_trim_manifest( assert acc not in remaining +@pytest.mark.s3 +def test_trim_manifest_missing_key_warns_without_raising( + mock_s3_client: botocore.client.BaseClient, # noqa: ARG001 + caplog: pytest.LogCaptureFixture, +) -> None: + """Missing transfer manifest in S3 logs a warning and returns without error.""" + with caplog.at_level(logging.WARNING): + _trim_manifest(PurePosixPath("manifests/missing.txt"), TEST_BUCKET, {"GCF_000001215.4"}) + assert "Manifest not found in S3" in caplog.text + assert [r for r in caplog.records if r.levelno >= logging.ERROR] == [] + + # Helpers for _archive_assemblies tests @@ -903,6 +924,39 @@ def test_archive_assemblies_invalid_accession_skipped( assert archived == 1 +@pytest.mark.s3 +def test_archive_assemblies_version_prefix_and_comments( + mock_s3_client: botocore.client.BaseClient, + tmp_path: Path, + caplog: pytest.LogCaptureFixture, +) -> None: + """Archiving GCF_000005845.2 skips '#' comments and does not touch GCF_000005845.20.""" + key_v2 = DEFAULT_DESTINATION_PREFIX / "raw_data" / ACC_PATH_845 / "GCF_000005845.2_genomic.fna.gz" + acc_path_v20 = PurePosixPath("GCF") / "000" / "005" / "845" / "GCF_000005845.20_ASM584v20" + key_v20 = DEFAULT_DESTINATION_PREFIX / "raw_data" / acc_path_v20 / "GCF_000005845.20_genomic.fna.gz" + mock_s3_client.put_object(Bucket=str(TEST_BUCKET), Key=str(key_v2), Body=b"v2") + mock_s3_client.put_object(Bucket=str(TEST_BUCKET), Key=str(key_v20), Body=b"v20") + + manifest = tmp_path / "removed.txt" + manifest.write_text("# comment header\n\nGCF_000005845.2\n", encoding="utf-8") + + with caplog.at_level(logging.WARNING): + archived = _archive_assemblies( + manifest, + lakehouse_bucket=TEST_BUCKET, + lakehouse_key_prefix=DEFAULT_DESTINATION_PREFIX, + ncbi_release="2024-01", + archive_reason="replaced_or_suppressed", + delete_source=True, + ) + + assert archived == 1 + assert "Invalid accession format" not in caplog.text + assert [r for r in caplog.records if r.levelno >= logging.ERROR] == [] + assert mock_s3_client.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(key_v2)).get("KeyCount", 0) == 0 + assert mock_s3_client.list_objects_v2(Bucket=str(TEST_BUCKET), Prefix=str(key_v20)).get("KeyCount", 0) == 1 + + # Concurrent / multi-file promotion (new behaviour) diff --git a/tests/cdm_data_loaders/datasets/ncbi_ftp/test_updates.py b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_updates.py new file mode 100644 index 00000000..d61522b3 --- /dev/null +++ b/tests/cdm_data_loaders/datasets/ncbi_ftp/test_updates.py @@ -0,0 +1,258 @@ +"""Unit tests for ``cdm_data_loaders.datasets.ncbi_ftp.updates``.""" + +from datetime import date +from pathlib import Path + +import pandas as pd +import pytest + +from cdm_data_loaders.datasets.ncbi_ftp.constants import Source +from cdm_data_loaders.datasets.ncbi_ftp.updates import ( + Period, + Splits, + batch_count, + build_interesting_membership_mask, + build_splits, + compute_batch_width, + filter_to_interesting, + infer_source, + load_ftp_mod_times, + load_interesting_assembly_ids, + parse_assembly_dir, + save_tsv, + split_by_period, + split_by_source, + write_id_batches, +) + + +@pytest.mark.parametrize( + ("assembly_dir", "expected"), + [ + pytest.param( + "GCF_000001215.4_Release_6_plus_ISO1_MT", + ("GCF_000001215.4", "Release_6_plus_ISO1_MT"), + id="gcf-multi-underscore-name", + ), + pytest.param( + "GCA_012345678.10_ASM123v10", + ("GCA_012345678.10", "ASM123v10"), + id="gca-two-digit-version", + ), + ], +) +def test_parse_assembly_dir_valid(assembly_dir: str, expected: tuple[str, str]) -> None: + """Split valid NCBI assembly directory names into accession and assembly name.""" + assert parse_assembly_dir(assembly_dir) == expected + + +@pytest.mark.parametrize( + "assembly_dir", + [ + pytest.param("GCF_000001215.4", id="missing-name-suffix"), + pytest.param("GCF_000001215.4_", id="empty-name-suffix"), + pytest.param("XYZ_000001215.4_Name", id="invalid-db-prefix"), + pytest.param("GCF_12345.1_ShortDigits", id="too-few-digits"), + ], +) +def test_parse_assembly_dir_invalid(assembly_dir: str) -> None: + """Raise ValueError when assembly_dir does not match ASSEMBLY_DIR_REGEX.""" + with pytest.raises(ValueError, match="does not match expected pattern"): + parse_assembly_dir(assembly_dir) + + +@pytest.mark.parametrize( + ("assembly_id", "expected"), + [ + pytest.param("GCF_000001215.4", Source.REFSEQ, id="refseq"), + pytest.param("GCA_012345678.1", Source.GENBANK, id="genbank"), + ], +) +def test_infer_source_valid(assembly_id: str, expected: Source) -> None: + """Map GCF_ and GCA_ prefixes to Source enum members.""" + assert infer_source(assembly_id) is expected + + +def test_infer_source_invalid() -> None: + """Raise ValueError when assembly_id lacks a GCF_ or GCA_ prefix.""" + with pytest.raises(ValueError, match="Unrecognized assembly_id prefix"): + infer_source("GBK_000001215.4") + + +def test_load_ftp_mod_times_and_interesting_ids(tmp_path: Path) -> None: + """Parse headerless modification-time TSV and interesting assembly ID list.""" + mod_file = tmp_path / "ftp_mod_times.txt" + mod_file.write_text( + "GCF_000001215.4_Rel6\t2024/01/15\nGCA_000005845.2_ASM584v2\t2024/06/01\n", + encoding="utf-8", + ) + ids_file = tmp_path / "assemblies.txt" + ids_file.write_text("GCA_000005845.2\nGCF_999999999.1\n", encoding="utf-8") + + df = load_ftp_mod_times(mod_file) + assert list(df.columns) == ["assembly_dir", "assembly_id", "assembly_name", "source", "modification_date"] + assert df.to_dict(orient="records") == [ + { + "assembly_dir": "GCF_000001215.4_Rel6", + "assembly_id": "GCF_000001215.4", + "assembly_name": "Rel6", + "source": Source.REFSEQ, + "modification_date": date(2024, 1, 15), + }, + { + "assembly_dir": "GCA_000005845.2_ASM584v2", + "assembly_id": "GCA_000005845.2", + "assembly_name": "ASM584v2", + "source": Source.GENBANK, + "modification_date": date(2024, 6, 1), + }, + ] + assert load_interesting_assembly_ids(ids_file) == {"GCA_000005845.2", "GCF_999999999.1"} + + +@pytest.mark.parametrize( + ("filter_refseq", "expected_ids"), + [ + pytest.param( + False, + ["GCA_000005845.2", "GCF_000001215.4", "GCF_000001405.40"], + id="keep-all-refseq", + ), + pytest.param( + True, + ["GCA_000005845.2", "GCF_000001215.4"], + id="filter-refseq-and-genbank", + ), + ], +) +def test_filter_to_interesting_deduplicates_latest_date( + filter_refseq: bool, + expected_ids: list[str], +) -> None: + """Filter rows according to filter_refseq and retain the latest modification date per assembly.""" + parsed_df = pd.DataFrame( + [ + { + "assembly_dir": "GCF_000001215.4_Old", + "assembly_id": "GCF_000001215.4", + "assembly_name": "Old", + "source": Source.REFSEQ, + "modification_date": date(2023, 1, 1), + }, + { + "assembly_dir": "GCF_000001215.4_New", + "assembly_id": "GCF_000001215.4", + "assembly_name": "New", + "source": Source.REFSEQ, + "modification_date": date(2024, 5, 10), + }, + { + "assembly_dir": "GCF_000001405.40_GRCh38", + "assembly_id": "GCF_000001405.40", + "assembly_name": "GRCh38", + "source": Source.REFSEQ, + "modification_date": date(2024, 2, 1), + }, + { + "assembly_dir": "GCA_000005845.2_ASM584v2", + "assembly_id": "GCA_000005845.2", + "assembly_name": "ASM584v2", + "source": Source.GENBANK, + "modification_date": date(2024, 3, 1), + }, + { + "assembly_dir": "GCA_000009999.1_DropMe", + "assembly_id": "GCA_000009999.1", + "assembly_name": "DropMe", + "source": Source.GENBANK, + "modification_date": date(2024, 4, 1), + }, + ] + ) + interesting_ids = {"GCF_000001215.4", "GCA_000005845.2"} + + mask = build_interesting_membership_mask(parsed_df, interesting_ids, filter_refseq=filter_refseq) + assert isinstance(mask, pd.Series) + + result = filter_to_interesting(parsed_df, interesting_ids, filter_refseq=filter_refseq) + assert result["assembly_id"].tolist() == expected_ids + rel_row = result.loc[result["assembly_id"] == "GCF_000001215.4"].iloc[0] + assert rel_row["assembly_name"] == "New" + assert rel_row["modification_date"] == date(2024, 5, 10) + + +def test_split_by_period_source_and_build_splits(tmp_path: Path) -> None: + """Partition assemblies across (Source, Period) with inclusive threshold date and save TSV.""" + interesting_df = pd.DataFrame( + [ + {"assembly_id": "GCF_000000001.1", "source": Source.REFSEQ, "modification_date": date(2024, 4, 30)}, + {"assembly_id": "GCF_000000002.1", "source": Source.REFSEQ, "modification_date": date(2024, 5, 1)}, + {"assembly_id": "GCA_000000003.1", "source": Source.GENBANK, "modification_date": date(2024, 5, 2)}, + ] + ) + threshold = date(2024, 5, 1) + + by_period = split_by_period(interesting_df, threshold) + assert by_period[Period.BEFORE]["assembly_id"].tolist() == ["GCF_000000001.1"] + assert by_period[Period.AFTER]["assembly_id"].tolist() == ["GCA_000000003.1", "GCF_000000002.1"] + + by_source = split_by_source(interesting_df) + assert by_source[Source.REFSEQ]["assembly_id"].tolist() == ["GCF_000000001.1", "GCF_000000002.1"] + assert by_source[Source.GENBANK]["assembly_id"].tolist() == ["GCA_000000003.1"] + + splits = build_splits(interesting_df, threshold) + assert {k: v["assembly_id"].tolist() for k, v in splits.items()} == { + (Source.REFSEQ, Period.BEFORE): ["GCF_000000001.1"], + (Source.REFSEQ, Period.AFTER): ["GCF_000000002.1"], + (Source.GENBANK, Period.BEFORE): [], + (Source.GENBANK, Period.AFTER): ["GCA_000000003.1"], + } + + tsv_path = tmp_path / "split.tsv" + save_tsv(splits[(Source.REFSEQ, Period.AFTER)], tsv_path) + assert tsv_path.read_text(encoding="utf-8") == ( + "assembly_id\tsource\tmodification_date\nGCF_000000002.1\trefseq\t2024-05-01\n" + ) + + +@pytest.mark.parametrize( + ("n_items", "batch_size", "expected"), + [ + pytest.param(0, 10, 0, id="empty"), + pytest.param(10, 10, 1, id="exact-batch"), + pytest.param(11, 10, 2, id="partial-remainder"), + ], +) +def test_batch_count(n_items: int, batch_size: int, expected: int) -> None: + """Compute ceiling batch counts with zero items returning zero batches.""" + assert batch_count(n_items, batch_size) == expected + + +@pytest.mark.parametrize( + ("row_counts", "batch_size", "expected_width"), + [ + pytest.param((0, 0), 5, 1, id="all-empty-splits"), + pytest.param((9, 2), 1, 1, id="single-digit-batches"), + pytest.param((25, 3), 2, 2, id="two-digit-batches"), + ], +) +def test_compute_batch_width(row_counts: tuple[int, int], batch_size: int, expected_width: int) -> None: + """Compute zero-padding digit width from the largest split batch count.""" + splits: Splits = { + (Source.REFSEQ, Period.AFTER): pd.DataFrame({"assembly_id": [f"id_{i}" for i in range(row_counts[0])]}), + (Source.GENBANK, Period.BEFORE): pd.DataFrame({"assembly_id": [f"id_{i}" for i in range(row_counts[1])]}), + } + assert compute_batch_width(splits, batch_size) == expected_width + + +def test_write_id_batches_writes_zero_padded_files(tmp_path: Path) -> None: + """Write numbered batch files with zero-padded suffixes and skip writing when ids is empty.""" + assert write_id_batches([], tmp_path, "empty_prefix", batch_size=2, width=2) == 0 + assert list(tmp_path.iterdir()) == [] + + ids = ["GCF_000000001.1", "GCF_000000002.1", "GCF_000000003.1"] + written = write_id_batches(ids, tmp_path, "after_20240501", batch_size=2, width=2) + expected_batches = 2 + assert written == expected_batches + assert (tmp_path / "after_20240501-01.txt").read_text(encoding="utf-8") == "GCF_000000001.1\nGCF_000000002.1\n" + assert (tmp_path / "after_20240501-02.txt").read_text(encoding="utf-8") == "GCF_000000003.1\n" diff --git a/tests/cdm_data_loaders/datasets/uniprot/test_common.py b/tests/cdm_data_loaders/datasets/uniprot/test_common.py new file mode 100644 index 00000000..25b990ce --- /dev/null +++ b/tests/cdm_data_loaders/datasets/uniprot/test_common.py @@ -0,0 +1,212 @@ +"""Unit tests for ``cdm_data_loaders.datasets.uniprot.common``.""" + +import datetime as dt +import json +import logging +from collections.abc import Mapping +from typing import Any + +import pytest + +from cdm_data_loaders.core.fields import REJECTED_TABLE_NAME +from cdm_data_loaders.core.registry import TransformContext +from cdm_data_loaders.core.transforms.introspection import inspect_transform_signature +from cdm_data_loaders.datasets.uniprot.common import ( + DEFAULT_ENTRY_TABLE_NAME, + NCBI_TAXON_PREFIX, + PREFIX_TRANSLATION, + UNIPROT_CURIE_PREFIX, + UNIREF_CURIE_PREFIX, + ensembl_type, + format_curie, + format_date, + iter_mappings, + make_cdm_entry_transform, + reject_cdm_entry, + resolve_timestamp, + run_cdm_entry_parser, +) + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + pytest.param({"a": 1}, [{"a": 1}], id="single-mapping"), + pytest.param([{"a": 1}, "skip", {"b": 2}], [{"a": 1}, {"b": 2}], id="sequence-filters-non-mappings"), + pytest.param("not-a-sequence-of-mappings", [], id="string-ignored"), + pytest.param(b"bytes-ignored", [], id="bytes-ignored"), + pytest.param(None, [], id="none-empty"), + ], +) +def test_iter_mappings_yields_only_mapping_items(value: object, expected: list[dict[str, Any]]) -> None: + """iter_mappings yields dicts from single mappings or sequences while ignoring strings, bytes, and None.""" + assert [dict(item) for item in iter_mappings(value)] == expected + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + pytest.param(dt.date(2025, 2, 5), "2025-02-05", id="date"), + pytest.param(dt.datetime(2025, 2, 5, 12, 30, tzinfo=dt.UTC), "2025-02-05T12:30:00+00:00", id="datetime"), + pytest.param("2025-02-05", "2025-02-05", id="string"), + pytest.param(12345, None, id="int-returns-none"), + pytest.param(None, None, id="none-returns-none"), + ], +) +def test_format_date_formats_dates_and_strings(value: object, expected: str | None) -> None: + """format_date returns ISO strings for dates and strings and None for other types.""" + assert format_date(value) == expected + + +@pytest.mark.parametrize( + ("xref", "expected"), + [ + pytest.param("ENST000001", "transcript", id="enst"), + pytest.param("ENSP000001", "protein sequence", id="ensp"), + pytest.param("ENSG000001", "gene sequence", id="ensg"), + pytest.param("ENSE000001", "exon", id="ense"), + pytest.param("ENSFM0001", "protein family", id="ensfm"), + pytest.param("ENSGT0001", "gene tree", id="ensgt"), + pytest.param("ENSR000001", "regulatory feature", id="ensr"), + pytest.param("OTHER123", "sequence", id="fallback-sequence"), + ], +) +def test_ensembl_type_infers_entity_category(xref: str, expected: str) -> None: + """ensembl_type maps 4- and 5-character Ensembl prefixes and falls back to sequence.""" + assert ensembl_type(xref) == expected + + +@pytest.mark.parametrize( + ("db", "accession", "fallback_upper", "expected"), + [ + pytest.param("UniProtKB", "P12345", False, "uniprot:P12345", id="uniprotkb"), + pytest.param("UniProtKB/Swiss-Prot", "P12345", False, "uniprot:P12345", id="swissprot"), + pytest.param("UniProtKB/TrEMBL", "A0A001", False, "uniprot:A0A001", id="trembl"), + pytest.param("UniProtKB accession", "P12345", False, "uniprot:P12345", id="uniprotkb-accession"), + pytest.param("UniProtKB ID", "ACE_HUMAN", False, "uniprot_name:ACE_HUMAN", id="uniprotkb-id"), + pytest.param("UniParc ID", "UPI0001", False, "uniparc:UPI0001", id="uniparc"), + pytest.param("UniRef50 ID", "UniRef50_P12345", False, "uniref:UniRef50_P12345", id="uniref50"), + pytest.param("UniRef90 ID", "UniRef90_P12345", False, "uniref:UniRef90_P12345", id="uniref90"), + pytest.param("UniRef100 ID", "UniRef100_P12345", False, "uniref:UniRef100_P12345", id="uniref100"), + pytest.param("NCBI Taxonomy", "9606", False, "NCBITaxon:9606", id="ncbi-taxonomy-upper-t"), + pytest.param("NCBI taxonomy", "9606", False, "NCBITaxon:9606", id="ncbi-taxonomy-lower-t"), + pytest.param("GeneID", "1636", False, "NCBIGene:1636", id="geneid"), + pytest.param("pubmed", "12345678", True, "PMID:12345678", id="pubmed-mapped"), + pytest.param("doi", "10.1000/x", True, "DOI:10.1000/x", id="unmapped-fallback-upper"), + pytest.param("CustomDB", "X1", False, "CustomDB:X1", id="unmapped-preserve-case"), + ], +) +def test_format_curie_translates_prefixes_and_applies_fallback( + db: str, + accession: str, + fallback_upper: bool, + expected: str, +) -> None: + """format_curie translates PREFIX_TRANSLATION keys and applies optional uppercase fallback.""" + assert UNIPROT_CURIE_PREFIX == "uniprot:" + assert UNIREF_CURIE_PREFIX == "uniref:" + assert NCBI_TAXON_PREFIX == "NCBITaxon" + assert "pubmed" in PREFIX_TRANSLATION + assert format_curie(db, accession, fallback_upper=fallback_upper) == expected + + +def test_resolve_timestamp_returns_explicit_or_utc_now() -> None: + """resolve_timestamp preserves an explicit timestamp and generates a UTC timestamp when None.""" + fixed = dt.datetime(2025, 3, 1, 8, 0, tzinfo=dt.UTC) + assert resolve_timestamp(fixed) == fixed + + generated = resolve_timestamp(None) + assert generated.tzinfo == dt.UTC + + +@pytest.mark.parametrize( + ("ctx", "default_table", "expected_table", "expected_source"), + [ + pytest.param(None, DEFAULT_ENTRY_TABLE_NAME, "entry", None, id="no-ctx-default-table"), + pytest.param(None, "custom_entry", "custom_entry", None, id="no-ctx-custom-default-table"), + pytest.param( + TransformContext(table_name="uniprot_entry", source_file="chunks/part_01.xml.gz"), + DEFAULT_ENTRY_TABLE_NAME, + "uniprot_entry", + "chunks/part_01.xml.gz", + id="with-ctx-table-and-source", + ), + ], +) +def test_reject_cdm_entry_preserves_context_provenance( + ctx: TransformContext | None, + default_table: str, + expected_table: str, + expected_source: str | None, +) -> None: + """reject_cdm_entry populates _rejected row with ctx.table_name and ctx.source_file when provided.""" + rejected = reject_cdm_entry({"_id": "bad"}, "missing id", ctx, default_table=default_table) + assert rejected == { + REJECTED_TABLE_NAME: [ + { + "table_name": expected_table, + "source_file": expected_source, + "line_no": None, + "raw_record": json.dumps({"_id": "bad"}), + "error_detail": "missing id", + } + ] + } + + +def test_run_cdm_entry_parser_returns_parsed_or_routes_exception_with_ctx( + caplog: pytest.LogCaptureFixture, +) -> None: + """run_cdm_entry_parser returns parse_fn output on success and routes exceptions to _rejected with ctx.""" + test_logger = logging.getLogger("test_uniprot_common") + ctx = TransformContext(table_name="entry", source_file="batch/in.xml.gz") + + ok = run_cdm_entry_parser( + {"_id": "ok"}, + lambda: {"entity": [{"entity_id": "uniprot:P1"}]}, + ctx=ctx, + logger=test_logger, + error_message="Unexpected parse failure", + ) + assert ok == {"entity": [{"entity_id": "uniprot:P1"}]} + + def _fail() -> dict[str, list[dict[str, Any]]]: + msg = "explode" + raise RuntimeError(msg) + + with caplog.at_level(logging.ERROR, logger="test_uniprot_common"): + failed = run_cdm_entry_parser( + {"_id": "broken"}, + _fail, + ctx=ctx, + logger=test_logger, + error_message="Unexpected parse failure", + ) + + assert "Unexpected parse failure" in caplog.text + assert failed[REJECTED_TABLE_NAME][0]["source_file"] == "batch/in.xml.gz" + assert failed[REJECTED_TABLE_NAME][0]["table_name"] == "entry" + assert failed[REJECTED_TABLE_NAME][0]["error_detail"] == "explode" + + +def test_make_cdm_entry_transform_supports_single_arg_and_context_invocation() -> None: + """make_cdm_entry_transform produces a context-aware RecordTransformFn callable with or without ctx.""" + + def _entry_fn(entry: Mapping[str, Any], ctx: TransformContext | None) -> dict[str, list[dict[str, Any]]]: + return { + "entity": [ + { + "id": entry["id"], + "source": ctx.source_file if ctx is not None else None, + } + ] + } + + transform_fn = make_cdm_entry_transform(_entry_fn) + _, _, takes_ctx = inspect_transform_signature(transform_fn) + + assert takes_ctx is True + assert transform_fn({"id": "P1"}) == {"entity": [{"id": "P1", "source": None}]} + assert transform_fn({"id": "P1"}, TransformContext(table_name="entry", source_file="chunk.xml")) == { + "entity": [{"id": "P1", "source": "chunk.xml"}] + } diff --git a/tests/cdm_data_loaders/datasets/uniprot/test_kb.py b/tests/cdm_data_loaders/datasets/uniprot/test_kb.py index 120d3d38..2d67922e 100644 --- a/tests/cdm_data_loaders/datasets/uniprot/test_kb.py +++ b/tests/cdm_data_loaders/datasets/uniprot/test_kb.py @@ -2,6 +2,7 @@ import datetime as dt import json +import logging from typing import Any import pytest @@ -10,6 +11,8 @@ import cdm_data_loaders.datasets.uniprot.kb as uniprot_kb_module from cdm_data_loaders.coercion.xml import make_xml_element_converter from cdm_data_loaders.core.fields import REJECTED_TABLE_NAME +from cdm_data_loaders.core.transforms.introspection import inspect_transform_signature +from cdm_data_loaders.core.transforms.routing import route_and_coerce_record from cdm_data_loaders.datasets.uniprot.kb import ( DB, DESCRIPTION, @@ -18,6 +21,7 @@ HTTPS_NS, NAME, NCBI_TAXON, + UNIPROT_CDM_SCHEMA_MAP, XREF, make_uniprot_cdm_transform, parse_cross_references, @@ -1070,6 +1074,40 @@ def test_make_uniprot_cdm_transform() -> None: ] +def test_make_uniprot_cdm_transform_propagates_source_file_to_rejected_and_logs_error( + caplog: pytest.LogCaptureFixture, +) -> None: + """make_uniprot_cdm_transform resolves a default UTC timestamp and preserves source_file on _rejected.""" + transform_fn = make_uniprot_cdm_transform() + assert inspect_transform_signature(transform_fn) == (False, True, True) + + valid_routed = transform_fn({"_dataset": "Swiss-Prot", "accession": ["P12345"], "name": ["TEST_HUMAN"]}) + updated = valid_routed["entity"][0]["updated"] + assert isinstance(updated, dt.datetime) + assert updated.tzinfo == dt.UTC + + with caplog.at_level(logging.ERROR, logger="cdm_data_loaders.datasets.uniprot.kb"): + routed = route_and_coerce_record( + {"_dataset": "Swiss-Prot", "accession": []}, + "entry", + transform=transform_fn, + schema_map=UNIPROT_CDM_SCHEMA_MAP, + source_file="chunks/uniprot_01.xml.gz", + ) + + assert "No accession found for entry" in caplog.text + assert list(routed.keys()) == [REJECTED_TABLE_NAME] + assert routed[REJECTED_TABLE_NAME] == [ + { + "table_name": "entry", + "source_file": "chunks/uniprot_01.xml.gz", + "line_no": None, + "raw_record": json.dumps({"_dataset": "Swiss-Prot", "accession": []}), + "error_detail": "No accession found", + } + ] + + FIXTURE_CHUNK = TEST_DATA_DIR / "uniprot" / "uniprot_kb" / "chunk_4" / "chunk_00001.xml" diff --git a/tests/cdm_data_loaders/datasets/uniprot/test_uniref.py b/tests/cdm_data_loaders/datasets/uniprot/test_uniref.py index c0afe99b..372c4aac 100644 --- a/tests/cdm_data_loaders/datasets/uniprot/test_uniref.py +++ b/tests/cdm_data_loaders/datasets/uniprot/test_uniref.py @@ -10,8 +10,12 @@ from cdm_data_loaders.coercion.xml import make_xml_element_converter from cdm_data_loaders.core.fields import REJECTED_TABLE_NAME +from cdm_data_loaders.core.registry import TransformContext +from cdm_data_loaders.core.transforms.introspection import inspect_transform_signature +from cdm_data_loaders.core.transforms.routing import route_and_coerce_record from cdm_data_loaders.datasets.uniprot.uniref import ( ENTRY_XML_TAG, + UNIREF_CDM_SCHEMA_MAP, UNIREF_URL, extract_cluster, extract_cross_refs, @@ -663,16 +667,18 @@ def test_extract_cluster_no_id() -> None: assert extract_cluster(CONVERT_ENTRY(fromstring(xml))) == ({}, {}) -def test_parse_uniref_entry_no_cluster_id() -> None: - """An entry with no id attribute is routed to _rejected rather than parsed.""" +def test_parse_uniref_entry_no_cluster_id(caplog: pytest.LogCaptureFixture) -> None: + """An entry with no id attribute logs an error and routes to _rejected.""" xml = f"Cluster: no id" entry_dict = CONVERT_ENTRY(fromstring(xml, parser=XMLParser(remove_blank_text=True))) - result = parse_uniref_entry( - entry_dict, - NOW, - PROTOCOL, - ) - + with caplog.at_level(logging.ERROR, logger="cdm_data_loaders.datasets.uniprot.uniref"): + result = parse_uniref_entry( + entry_dict, + NOW, + PROTOCOL, + ) + + assert "No cluster ID found in entry" in caplog.text assert list(result.keys()) == [REJECTED_TABLE_NAME] error = result[REJECTED_TABLE_NAME][0] assert error["table_name"] == "entry" @@ -682,6 +688,26 @@ def test_parse_uniref_entry_no_cluster_id() -> None: assert json.loads(error["raw_record"]) == {"name": "Cluster: no id"} +def test_parse_uniref_entry_unexpected_error(caplog: pytest.LogCaptureFixture) -> None: + """Unexpected parse exceptions log at ERROR and route to _rejected with TransformContext provenance.""" + malformed_entry: dict[str, Any] = { + "_id": "UniRef50_BAD", + "representativeMember": {"dbReference": {"_type": ["unhashable"], "_id": "UPI0001"}}, + } + ctx = TransformContext(table_name="entry", source_file="chunks/uniref_01.xml.gz") + with caplog.at_level(logging.ERROR, logger="cdm_data_loaders.datasets.uniprot.uniref"): + result = parse_uniref_entry(malformed_entry, NOW, PROTOCOL, ctx=ctx) + + assert "Parse error while processing UniRef entry" in caplog.text + assert list(result.keys()) == [REJECTED_TABLE_NAME] + error = result[REJECTED_TABLE_NAME][0] + assert error["table_name"] == "entry" + assert error["source_file"] == "chunks/uniref_01.xml.gz" + assert error["line_no"] is None + assert "unhashable" in error["error_detail"] + assert json.loads(error["raw_record"]) == malformed_entry + + def test_make_uniref_cdm_transform() -> None: """Test that make_uniref_cdm_transform builds a RecordTransformFn that routes UniRef entries.""" transform_fn = make_uniref_cdm_transform(PROTOCOL, timestamp=NOW) @@ -729,6 +755,40 @@ def test_make_uniref_cdm_transform() -> None: ] +def test_make_uniref_cdm_transform_propagates_source_file_to_rejected( + caplog: pytest.LogCaptureFixture, +) -> None: + """make_uniref_cdm_transform resolves a default UTC timestamp and preserves source_file on _rejected.""" + transform_fn = make_uniref_cdm_transform(PROTOCOL) + assert inspect_transform_signature(transform_fn) == (False, True, True) + + valid_routed = transform_fn({"_id": "UniRef50_TEST", "name": "Cluster: Test"}) + updated = valid_routed["entity"][0]["updated"] + assert isinstance(updated, dt.datetime) + assert updated.tzinfo == dt.UTC + + with caplog.at_level(logging.ERROR, logger="cdm_data_loaders.datasets.uniprot.uniref"): + routed = route_and_coerce_record( + {"name": "Cluster: missing id"}, + "entry", + transform=transform_fn, + schema_map=UNIREF_CDM_SCHEMA_MAP, + source_file="chunks/uniref50_01.xml.gz", + ) + + assert "No cluster ID found in entry" in caplog.text + assert list(routed.keys()) == [REJECTED_TABLE_NAME] + assert routed[REJECTED_TABLE_NAME] == [ + { + "table_name": "entry", + "source_file": "chunks/uniref50_01.xml.gz", + "line_no": None, + "raw_record": json.dumps({"name": "Cluster: missing id"}), + "error_detail": "No cluster ID found in entry", + } + ] + + FIXTURE_CHUNK = TEST_DATA_DIR / "uniprot" / "uniref" / "integration" / "uniref_chunk_00001.xml" diff --git a/tests/cdm_data_loaders/io/s3/test_object_utils.py b/tests/cdm_data_loaders/io/s3/test_object_utils.py index 23e21c35..5a88acd0 100644 --- a/tests/cdm_data_loaders/io/s3/test_object_utils.py +++ b/tests/cdm_data_loaders/io/s3/test_object_utils.py @@ -21,7 +21,9 @@ download_file, get_existing_object_info, get_object_bytes, + get_object_bytes_if_exists, head_object, + is_not_found_client_error, list_objects, object_exists, put_object_bytes, @@ -186,9 +188,13 @@ def test_get_existing_object_info_pass_returns_info_for_existing_object( @pytest.mark.s3 @pytest.mark.usefixtures("mock_s3_client") -def test_get_existing_object_info_pass_returns_none_for_missing_object() -> None: - """Verify None is returned (not an exception) for a missing object.""" - assert get_existing_object_info(f"{TEST_BUCKET}does/not/exist.txt") is None +def test_get_existing_object_info_pass_returns_none_for_missing_object( + caplog: pytest.LogCaptureFixture, +) -> None: + """Verify None is returned without logging ERROR records for a missing object.""" + with caplog.at_level(logging.ERROR): + assert get_existing_object_info(f"{TEST_BUCKET}does/not/exist.txt") is None + assert caplog.records == [] @pytest.mark.s3 @@ -1347,3 +1353,53 @@ def test_put_object_bytes_defaults_without_optional_headers() -> None: s3_path = f"{TEST_BUCKET}/bytes/raw.bin" put_object_bytes(b"raw-bytes", s3_path) assert get_object_bytes(s3_path) == b"raw-bytes" + + +@pytest.mark.parametrize( + ("code", "status", "expected"), + [ + pytest.param("NoSuchKey", 404, True, id="no-such-key"), + pytest.param("NoSuchBucket", 400, True, id="no-such-bucket"), + pytest.param("CustomCode", 404, True, id="404-with-custom-code"), + pytest.param("AccessDenied", 403, False, id="access-denied"), + pytest.param("InternalError", 500, False, id="internal-error"), + ], +) +def test_is_not_found_client_error(code: str, status: int, expected: bool) -> None: + """Classify S3 ClientError instances into not-found vs other errors.""" + exc = ClientError({"Error": {"Code": code}, "ResponseMetadata": {"HTTPStatusCode": status}}, "GetObject") + assert is_not_found_client_error(exc) is expected + + +@pytest.mark.s3 +def test_get_object_bytes_if_exists_existing_and_missing( + mock_s3_client: S3Client, + caplog: pytest.LogCaptureFixture, +) -> None: + """Return payload bytes when present and None without ERROR logs when absent.""" + mock_s3_client.put_object(Bucket=TEST_BUCKET, Key="present/data.bin", Body=b"present-bytes") + assert get_object_bytes_if_exists(f"{TEST_BUCKET}/present/data.bin") == b"present-bytes" + + with caplog.at_level(logging.ERROR): + assert get_object_bytes_if_exists(f"{TEST_BUCKET}/missing/data.bin") is None + assert caplog.records == [] + + +@pytest.mark.s3 +def test_get_object_bytes_if_exists_non_404_logs_and_reraises( + mock_s3_client: S3Client, + monkeypatch: pytest.MonkeyPatch, + caplog: pytest.LogCaptureFixture, +) -> None: + """Log an ERROR record and re-raise when S3 returns a non-not-found ClientError.""" + + def raise_access_denied(**_kwargs: object) -> None: + raise ClientError( + {"Error": {"Code": "AccessDenied"}, "ResponseMetadata": {"HTTPStatusCode": 403}}, + "GetObject", + ) + + monkeypatch.setattr(mock_s3_client, "get_object", raise_access_denied) + with caplog.at_level(logging.ERROR), pytest.raises(ClientError, match="AccessDenied"): + get_object_bytes_if_exists(f"{TEST_BUCKET}/denied/data.bin") + assert any("Error reading object" in r.getMessage() for r in caplog.records) diff --git a/tests/cdm_data_loaders/io/test_async_client.py b/tests/cdm_data_loaders/io/test_async_client.py index 3cdcf87a..6ee51f1e 100644 --- a/tests/cdm_data_loaders/io/test_async_client.py +++ b/tests/cdm_data_loaders/io/test_async_client.py @@ -87,4 +87,4 @@ async def test_async_client_unreachable_code(monkeypatch: pytest.MonkeyPatch) -> monkeypatch.setattr(downloader, "_retry", EmptyAsyncRetry()) with pytest.raises(RuntimeError, match="Iterator exhausted, unreachable code reached!"): - await downloader._download_with_retry(DOWNLOAD_URL, "file.txt") # noqa: SLF001 + await downloader._download_with_retry(DOWNLOAD_URL, Path("file.txt"), None, "sha256", {}) # noqa: SLF001 diff --git a/tests/cdm_data_loaders/io/test_clients.py b/tests/cdm_data_loaders/io/test_clients.py index 2db19a50..a7039cb0 100644 --- a/tests/cdm_data_loaders/io/test_clients.py +++ b/tests/cdm_data_loaders/io/test_clients.py @@ -13,6 +13,7 @@ DownloadError, NonRetryableDownloadError, ) +from cdm_data_loaders.io.sync_client import FileDownloader, fetch_json from tests.cdm_data_loaders.io._helpers import DownloaderAdapter DOWNLOAD_URL = "https://example.com/file.txt" @@ -285,10 +286,15 @@ def handler_timeout(_: httpx.Request) -> httpx.Response: # check the logs: we should have messages for each error/timeout until the successful download retry_possible_msgs = [r for r in caplog.records if r.getMessage().endswith("retry possible")] + before_sleep_msgs = [ + r for r in caplog.records if r.levelno == logging.WARNING and r.getMessage().startswith("Retrying ") + ] if max_attempts < successful_attempt: assert len(retry_possible_msgs) == max_attempts + assert len(before_sleep_msgs) == max_attempts - 1 else: assert len(retry_possible_msgs) == successful_attempt - 1 + assert len(before_sleep_msgs) == successful_attempt - 1 @patch("tenacity.nap.time.sleep", MagicMock()) @@ -319,3 +325,70 @@ def handler(_: httpx.Request) -> httpx.Response: ) # there will only ever be one retry assert calls["n"] == 1 + + +@patch("tenacity.nap.time.sleep", MagicMock()) +@pytest.mark.parametrize( + "failure_kind", + [ + pytest.param("503", id="server-503"), + pytest.param("timeout", id="connect-timeout"), + ], +) +def test_file_downloader_fetch_json_retries_transient_errors(failure_kind: str) -> None: + """Retry transient 5xx and timeout failures in FileDownloader.fetch_json and return decoded JSON.""" + attempts = 0 + expected_calls = 2 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + assert request.headers["accept"] == "application/json" + if attempts == 1: + if failure_kind == "503": + return httpx.Response(503) + err = "timeout" + raise httpx.ConnectTimeout(err) + return httpx.Response(200, json={"data": {"id": "r6gcp"}}) + + downloader = FileDownloader(httpx.Client(transport=httpx.MockTransport(handler)), max_attempts=3) + payload = downloader.fetch_json(DOWNLOAD_URL, extra_headers={"accept": "application/json"}) + assert payload == {"data": {"id": "r6gcp"}} + assert attempts == expected_calls + + +def test_file_downloader_fetch_json_fails_fast_on_404() -> None: + """Raise NonRetryableDownloadError on 404 without retrying.""" + attempts = 0 + + def handler(_request: httpx.Request) -> httpx.Response: + nonlocal attempts + attempts += 1 + return httpx.Response(404) + + downloader = FileDownloader(httpx.Client(transport=httpx.MockTransport(handler)), max_attempts=3) + with pytest.raises(NonRetryableDownloadError, match="Client error: 404 Not Found"): + downloader.fetch_json(DOWNLOAD_URL) + assert attempts == 1 + + +def test_module_fetch_json_applies_timeout_and_headers(monkeypatch: pytest.MonkeyPatch) -> None: + """Construct a managed httpx.Client with custom timeout and return the JSON payload.""" + captured_timeout: list[httpx.Timeout] = [] + + class FakeClient(httpx.Client): + def __init__(self, **kwargs: object) -> None: + timeout = kwargs.get("timeout") + if isinstance(timeout, httpx.Timeout): + captured_timeout.append(timeout) + super().__init__( + transport=httpx.MockTransport( + lambda req: httpx.Response(200, json={"accept": req.headers.get("accept")}) + ) + ) + + monkeypatch.setattr("cdm_data_loaders.io.sync_client.httpx.Client", FakeClient) + custom_timeout = 12.5 + result = fetch_json(DOWNLOAD_URL, timeout=custom_timeout, extra_headers={"accept": "application/json"}) + assert result == {"accept": "application/json"} + assert captured_timeout == [httpx.Timeout(custom_timeout)] diff --git a/tests/cdm_data_loaders/io/test_ftp_client.py b/tests/cdm_data_loaders/io/test_ftp_client.py index 164d47e8..a515c956 100644 --- a/tests/cdm_data_loaders/io/test_ftp_client.py +++ b/tests/cdm_data_loaders/io/test_ftp_client.py @@ -17,7 +17,9 @@ ftp_list_dir, ftp_noop_keepalive, ftp_retrieve_text, + ftp_session, ) +from cdm_data_loaders.utils.checksums import compute_file_checksum IDLE_SECONDS = 30 KEEPIDLE_VALUE = 60 @@ -90,8 +92,18 @@ def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: mock_ftp.cwd.assert_called_once_with("/some/path") -def test_retries_on_error_temp() -> None: - """Verify retry logic on FTP temporary errors.""" +@pytest.mark.parametrize( + "exc_cls", + [ + pytest.param(error_temp, id="error_temp"), + pytest.param(BrokenPipeError, id="broken_pipe"), + pytest.param(ConnectionResetError, id="connection_reset"), + pytest.param(EOFError, id="eof"), + ], +) +@patch("tenacity.nap.time.sleep", MagicMock()) +def test_retries_on_error_temp(exc_cls: type[BaseException]) -> None: + """Verify retry logic on transient FTP and socket errors.""" mock_ftp = MagicMock(spec=FTP) call_count = 0 @@ -99,7 +111,7 @@ def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: nonlocal call_count call_count += 1 if call_count < EXPECTED_RETRY_COUNT: - raise error_temp(ERR_421) # noqa: S321 + raise exc_cls(ERR_421) callback("file.txt") mock_ftp.retrlines.side_effect = fake_retrlines @@ -161,6 +173,27 @@ def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: assert result == "line1\nline2" +@patch("tenacity.nap.time.sleep", MagicMock()) +def test_retrieve_text_retries_and_discards_partial_lines() -> None: + """Verify ftp_retrieve_text retries transient errors and discards partial lines.""" + mock_ftp = MagicMock(spec=FTP) + call_count = 0 + + def fake_retrlines(_cmd: str, callback: Callable[[str], None]) -> None: + nonlocal call_count + call_count += 1 + if call_count < EXPECTED_RETRY_COUNT: + callback("stale_partial_line") + raise ConnectionResetError + for line in ["fresh1", "fresh2"]: + callback(line) + + mock_ftp.retrlines.side_effect = fake_retrlines + result = ftp_retrieve_text(mock_ftp, PurePosixPath("remote.txt")) + assert result == "fresh1\nfresh2" + assert call_count == EXPECTED_RETRY_COUNT + + @patch("cdm_data_loaders.io.ftp_client.connect_ftp") def test_get_returns_same_connection(mock_connect: MagicMock) -> None: """Verify get() returns the same FTP connection on repeated calls.""" @@ -182,3 +215,60 @@ def test_close_all(mock_connect: MagicMock) -> None: pool.get() pool.close_all() mock_ftp.quit.assert_called_once() + + +@patch("cdm_data_loaders.io.ftp_client.connect_ftp") +def test_ftp_session_owned_vs_borrowed_lifecycle(mock_connect: MagicMock) -> None: + """Close FTP connection on exit when owned, and leave caller-supplied FTP open.""" + owned_ftp = MagicMock(spec=FTP) + mock_connect.return_value = owned_ftp + with ftp_session("ftp.example.com", timeout=FTP_TIMEOUT) as active: + assert active is owned_ftp + mock_connect.assert_called_once_with("ftp.example.com", timeout=FTP_TIMEOUT) + owned_ftp.quit.assert_called_once() + + borrowed_ftp = MagicMock(spec=FTP) + with ftp_session("ftp.example.com", ftp=borrowed_ftp) as active: + assert active is borrowed_ftp + borrowed_ftp.quit.assert_not_called() + + +@pytest.mark.parametrize( + "checksum_fn", + [ + pytest.param("md5", id="md5"), + pytest.param("crc64nvme", id="crc64nvme"), + ], +) +@patch("tenacity.nap.time.sleep", MagicMock()) +def test_ftp_download_file_in_stream_checksum_resets_on_retry( + tmp_path: Path, + checksum_fn: str, +) -> None: + """Compute in-stream digest matching only the final successful transfer after a partial-write failure.""" + mock_ftp = MagicMock(spec=FTP) + call_count = 0 + + def fake_retrbinary(_cmd: str, callback: Callable[[bytes], None]) -> None: + nonlocal call_count + call_count += 1 + if call_count == 1: + callback(b"aborted-partial-bytes-") + raise BrokenPipeError + callback(b"final-") + callback(b"payload") + + mock_ftp.retrbinary.side_effect = fake_retrbinary + local = tmp_path / "streamed.bin" + digest = ftp_download_file(mock_ftp, PurePosixPath("remote.bin"), local, retries=3, checksum_fn=checksum_fn) + + assert local.read_bytes() == b"final-payload" + assert digest == compute_file_checksum(local, checksum_fn) + + +def test_ftp_download_file_invalid_checksum_fn_raises_before_transfer(tmp_path: Path) -> None: + """Reject unsupported checksum algorithm names before issuing FTP RETR.""" + mock_ftp = MagicMock(spec=FTP) + with pytest.raises(ValueError, match="not supported"): + ftp_download_file(mock_ftp, PurePosixPath("remote.bin"), tmp_path / "out.bin", checksum_fn="not_an_algo") + mock_ftp.retrbinary.assert_not_called() diff --git a/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py b/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py index 64f479ce..df566b76 100644 --- a/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py +++ b/tests/cdm_data_loaders/pipelines/all_the_bacteria/test_all_the_bacteria.py @@ -13,7 +13,6 @@ import pytest from frozendict import frozendict from pydantic import ValidationError -from requests.exceptions import HTTPError import cdm_data_loaders.datasets.all_the_bacteria.osf as atb_osf from cdm_data_loaders.core.cli import CliRunner @@ -66,7 +65,7 @@ def test_download_atb_index_tsv_vcr_destination_s3(test_s3_settings: AtbSettings @pytest.mark.vcr def test_download_atb_index_tsv_error_404(test_settings: AtbSettings) -> None: """Ensure that a 404 response causes an error and the function to die.""" - with pytest.raises(HTTPError, match="404 Client Error"): + with pytest.raises(NonRetryableDownloadError, match="Client error: 404 Not Found"): download_atb_index_tsv(test_settings) diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py index a6de3f41..340d917b 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_base.py @@ -162,23 +162,37 @@ def test_build_entity_resource_pass_no_matching_files( @pytest.mark.parametrize( - ("pipeline_run_kwargs", "run_pipeline_return", "expected_run_kwargs"), + ("cli_prog_name", "pipeline_run_kwargs", "run_pipeline_return", "expected_pipeline_name", "expected_run_kwargs"), [ pytest.param( + "jsonlines_pydantic", None, MagicMock(spec=LoadInfo), + "jsonlines_pydantic", {"loader_file_format": "parquet"}, - id="default_pipeline_run_kwargs", + id="uses_settings_cli_prog_name_with_default_run_kwargs", ), pytest.param( + None, + None, + MagicMock(spec=LoadInfo), + "test_registry_pipeline", + {"loader_file_format": "parquet"}, + id="falls_back_to_pipeline_name_when_cli_prog_name_unset", + ), + pytest.param( + "jsonlines_pydantic", {"loader_file_format": "jsonl", "schema_contract": "freeze"}, MagicMock(spec=LoadInfo), + "jsonlines_pydantic", {"loader_file_format": "jsonl", "schema_contract": "freeze"}, id="merges_and_overrides_pipeline_run_kwargs", ), pytest.param( + "jsonlines_pydantic", None, None, + "jsonlines_pydantic", {"loader_file_format": "parquet"}, id="propagates_none_on_pipeline_failure", ), @@ -187,12 +201,21 @@ def test_build_entity_resource_pass_no_matching_files( def test_run_registry_ingest_pass_builds_sorted_resources_and_delegates_to_run_pipeline( settings_factory: Callable[..., JsonlPydanticIngestSettings], widget_model: type[BaseModel], + cli_prog_name: str | None, pipeline_run_kwargs: dict[str, Any] | None, run_pipeline_return: LoadInfo | None, + expected_pipeline_name: str, expected_run_kwargs: dict[str, Any], ) -> None: """run_registry_ingest builds resources in sorted table order and delegates to core.run_pipeline.""" - settings = settings_factory(dataset_name="registry_ds", loader_file_format="parquet") + + class CustomProgSettings(JsonlPydanticIngestSettings, cli_prog_name=cli_prog_name): + pass + + base_settings = settings_factory(dataset_name="registry_ds", loader_file_format="parquet") + settings = CustomProgSettings.model_validate( + base_settings.model_dump(exclude=set(JsonlPydanticIngestSettings.model_computed_fields)) + ) registry: dict[str, type[BaseModel]] = { "zeta": widget_model, "alpha": widget_model, @@ -224,7 +247,7 @@ def fake_build_resource(table_name: str, entry: Any, cfg: Any) -> DltResource: settings=settings, resource=[built_resources["alpha"], built_resources["zeta"]], pipeline_kwargs={ - "pipeline_name": "test_registry_pipeline", + "pipeline_name": expected_pipeline_name, "dataset_name": "registry_ds", }, pipeline_run_kwargs=expected_run_kwargs, diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py index 2cc342a9..3b5a4b4f 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_extract_pipeline.py @@ -286,3 +286,18 @@ def fail_transform(_row: dict[str, Any]) -> dict[str, list[dict[str, Any]]]: "input_dir_files": ["nested/notes.txt", "nested/part.jsonl.gz", "top.jsonl"], "matched_files": ["nested/part.jsonl.gz", "top.jsonl"], } + + +def test_run_jsonlines_pipeline_uses_subclass_cli_prog_name( + extract_settings_factory: Callable[..., JsonlExtractSettings], +) -> None: + """run_jsonlines_pipeline forwards a preset subclass cli_prog_name as pipeline_name.""" + + class PresetExtractSettings(JsonlExtractSettings, cli_prog_name="preset_extract"): + pass + + base_settings = extract_settings_factory() + preset_settings = PresetExtractSettings.model_validate( + base_settings.model_dump(exclude=set(JsonlExtractSettings.model_computed_fields)) + ) + assert_core_run_pipeline_args(extract_pipeline_module, run_jsonlines_pipeline, preset_settings, "preset_extract") diff --git a/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py b/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py index 20c5440a..dc0c94de 100644 --- a/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/jsonlines/test_pydantic_pipeline.py @@ -424,3 +424,22 @@ def reject_corrupt_file(file_item: Any) -> Any: # noqa: ANN401 assert row["table_name"] == "widget" assert row["source_file"] == "widget/corrupt.jsonl" assert row["error_detail"] == "corrupt file header" + + +def test_run_jsonlines_with_validation_pipeline_uses_subclass_cli_prog_name( + scenario_input_dir: Callable[[str], str], + settings_factory: Callable[..., JsonlPydanticIngestSettings], + tmp_path: Path, +) -> None: + """run_jsonlines_with_validation_pipeline uses a preset subclass cli_prog_name as pipeline_name.""" + + class PresetPydanticSettings(JsonlPydanticIngestSettings, cli_prog_name="preset_pydantic"): + pass + + base_settings = settings_factory(input_dir=scenario_input_dir("happy_path"), output_dir=str(tmp_path / "out")) + preset_settings = PresetPydanticSettings.model_validate( + base_settings.model_dump(exclude=set(JsonlPydanticIngestSettings.model_computed_fields)) + ) + load_info = assert_run_ok(run_jsonlines_with_validation_pipeline(preset_settings)) + + assert load_info.pipeline.pipeline_name == "preset_pydantic" diff --git a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py index 3c562529..0fb72485 100644 --- a/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py +++ b/tests/cdm_data_loaders/pipelines/xmltodict/test_xmltodict_pipeline.py @@ -150,3 +150,18 @@ def transform(row: dict) -> dict: ("book", [{"book_id": "2", "title": "The Stand"}]), ("book_id_only", [{"book_id": "2"}]), ] + + +def test_run_xmltodict_pipeline_uses_subclass_cli_prog_name( + settings_factory: Callable[..., XmlToDictSettings], +) -> None: + """run_xmltodict_pipeline forwards a preset subclass cli_prog_name as pipeline_name.""" + + class PresetXmlToDictSettings(XmlToDictSettings, cli_prog_name="preset_xmltodict"): + pass + + base_settings = settings_factory() + preset_settings = PresetXmlToDictSettings.model_validate( + base_settings.model_dump(exclude=set(XmlToDictSettings.model_computed_fields)) + ) + assert_core_run_pipeline_args(xmltodict_module, run_xmltodict_pipeline, preset_settings, "preset_xmltodict") diff --git a/tests/cdm_data_loaders/utils/test_arrow_schema.py b/tests/cdm_data_loaders/utils/test_arrow_schema.py index 472c838b..ee4979c2 100644 --- a/tests/cdm_data_loaders/utils/test_arrow_schema.py +++ b/tests/cdm_data_loaders/utils/test_arrow_schema.py @@ -7,12 +7,14 @@ import pytest from cdm_data_loaders.utils.arrow_schema import ( + SCHEMA_COMMENT_METADATA_KEY, assemble_pyarrow_schema, decode_field_metadata, is_any_binary_type, is_any_list_type, is_any_string_type, is_non_json_arrow_type, + with_schema_description, ) SAMPLE_FIELDS: Final[tuple[pa.Field, ...]] = ( @@ -118,3 +120,40 @@ def test_arrow_type_predicates_classify_binary_string_and_list_variants( def test_is_non_json_arrow_type(dtype: pa.DataType, expected: bool) -> None: """Detect temporal, decimal, and binary Arrow types that cannot serialize with json.dumps.""" assert is_non_json_arrow_type(dtype) is expected + + +@pytest.mark.parametrize( + ("initial_metadata", "expected_metadata"), + [ + pytest.param( + None, + {SCHEMA_COMMENT_METADATA_KEY: b"Protein identifier mapping."}, + id="no-prior-metadata", + ), + pytest.param( + {b"source": b"unit"}, + {b"source": b"unit", SCHEMA_COMMENT_METADATA_KEY: b"Protein identifier mapping."}, + id="preserves-existing-keys", + ), + pytest.param( + {SCHEMA_COMMENT_METADATA_KEY: b"stale", b"keep": b"yes"}, + {SCHEMA_COMMENT_METADATA_KEY: b"Protein identifier mapping.", b"keep": b"yes"}, + id="replaces-existing-comment", + ), + ], +) +def test_with_schema_description_attaches_comment_and_preserves_metadata( + initial_metadata: dict[bytes, bytes] | None, + expected_metadata: dict[bytes, bytes], +) -> None: + """Attach UTF-8 comment metadata to a PyArrow schema without mutating the input schema.""" + base_schema = pa.schema( + [pa.field("id", pa.string(), nullable=False, metadata={b"definition": b"ID"})], + metadata=initial_metadata, + ) + updated = with_schema_description(base_schema, "Protein identifier mapping.") + + assert SCHEMA_COMMENT_METADATA_KEY == b"comment" + assert updated.metadata == expected_metadata + assert updated.field("id").metadata == {b"definition": b"ID"} + assert base_schema.metadata == initial_metadata diff --git a/tests/cdm_data_loaders/utils/test_checksums.py b/tests/cdm_data_loaders/utils/test_checksums.py index d74ca845..78ff6c88 100644 --- a/tests/cdm_data_loaders/utils/test_checksums.py +++ b/tests/cdm_data_loaders/utils/test_checksums.py @@ -21,6 +21,7 @@ Hasher, HashingReader, compute_file_checksum, + finalize_digest, new_hasher, parse_md5_checksums_file, parse_md5_sidecar, @@ -592,3 +593,16 @@ def test_parse_md5_checksums_file_parses_gnu_manifest() -> None: PurePosixPath("GCF_000001405_genomic.fna.gz"): "d41d8cd98f00b204e9800998ecf8427e", PurePosixPath("annotation_hashes.txt"): "098f6bcd4621d373cade4e832627b4f6", } + + +def test_finalize_digest_formats_hex_and_crc64nvme_base64() -> None: + """Return hex digests for hashlib hashers and base64 digests for Crc64NvmeHasher.""" + payload = b"streamed-payload" + sha_hasher = new_hasher("sha256") + sha_hasher.update(payload) + assert finalize_digest(sha_hasher) == hashlib.sha256(payload).hexdigest() + + crc_hasher = new_hasher("crc64nvme") + crc_hasher.update(payload) + expected_crc = _crc64nvme(payload, 0) + assert finalize_digest(crc_hasher) == base64.b64encode(expected_crc.to_bytes(8, "big")).decode()