Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 22 additions & 17 deletions docs/table_transforms.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ All six file-format ingestion pipelines support pluggable file pre-processing, r
| [`jsonlines/pydantic.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/jsonlines/pydantic.py) | `jsonlines_pydantic` | `pyarrow.Table` | Keys of `ENTITY_MODELS` |
| [`xsv/pipeline.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/xsv/pipeline.py) | `xsv` | `pyarrow.Table` | `--table-name` (or preset entrypoint `uniprot_idmapping`) |
| [`xmltodict/pipeline.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/xmltodict/pipeline.py) | `xmltodict` | `list[dict]` (`pyarrow.Table` if schema map set) | `--table-name` (`settings.table_name`) |
| [`xmltopyarrow/pipeline.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py) | `xmltopyarrow` | `pyarrow.Table` | `--table-name` (or preset entrypoints `pubmed`, `uniprot`, `uniref50`, `uniref90`, `uniref100`) |
| [`xmltopyarrow/pipeline.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/pipelines/xmltopyarrow/pipeline.py) | `xmltopyarrow` | `pyarrow.Table` | `--table-name` (or preset entrypoints `pubmed`, `uniprot`, `uniref50`, `uniref90`, `uniref100`, `uniprot_cdm`, `uniref50_cdm`, `uniref90_cdm`, `uniref100_cdm`) |

---

Expand Down Expand Up @@ -65,7 +65,7 @@ Create an importable Python module exporting `FILE_PREPROCESSORS`, `TABLE_TRANSF

**`OUTPUT_SCHEMAS`** (`--output-schemas-module` / `CDL_OUTPUT_SCHEMAS_MODULE`): `dict[str, pa.Schema]`

- Optional mapping of `{destination_table_name: pyarrow.Schema}` overriding or supplying `pyarrow.Schema` definitions for transformed or split destination tables. Preset settings subclasses can also return `OutputTableHints` from `get_output_table_hints()` ([`src/cdm_data_loaders/core/transforms/hints.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/core/transforms/hints.py)) to register table/column descriptions and Iceberg partition specs on `DltResource` variants.
- Optional mapping of `{destination_table_name: pyarrow.Schema}` overriding or supplying `pyarrow.Schema` definitions for transformed or split destination tables. Preset settings subclasses can lock built-in transforms and output schemas off the CLI with `LockedTransformHooksMixin` ([`src/cdm_data_loaders/core/transforms/settings_mixin.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/core/transforms/settings_mixin.py)), attach table descriptions via `with_schema_description` ([`src/cdm_data_loaders/utils/arrow_schema.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/utils/arrow_schema.py)), and return `OutputTableHints` from `get_output_table_hints()` ([`src/cdm_data_loaders/core/transforms/hints.py`](https://github.com/kbase/cdm-data-loaders/blob/develop/src/cdm_data_loaders/core/transforms/hints.py)) to register table/column descriptions and Iceberg partition specs on `DltResource` variants.

---

Expand Down Expand Up @@ -488,23 +488,28 @@ import pyarrow.compute as pc

from cdm_data_loaders.core.registry import TableTransformFn, TransformContext
from cdm_data_loaders.core.transforms.routing import emit_raw_table

RAW_ID_MAPPING_SCHEMA = pa.schema(
[
pa.field("uniprot_id", pa.string(), nullable=False),
pa.field("db_reference_type", pa.string(), nullable=False),
pa.field("db_reference_id", pa.string(), nullable=False),
],
metadata={"comment": "raw ID mapping data from UniProt"},
from cdm_data_loaders.utils.arrow_schema import with_schema_description

RAW_ID_MAPPING_SCHEMA = with_schema_description(
pa.schema(
[
pa.field("uniprot_id", pa.string(), nullable=False),
pa.field("db_reference_type", pa.string(), nullable=False),
pa.field("db_reference_id", pa.string(), nullable=False),
]
),
"raw ID mapping data from UniProt",
)

ID_MAPPING_SCHEMA = pa.schema(
[
pa.field("uniprot_id", pa.string(), nullable=False),
pa.field("db_reference", pa.string(), nullable=False),
pa.field("source", pa.string(), nullable=False),
],
metadata={"comment": "Standardised mapping data from UniProt"},
ID_MAPPING_SCHEMA = with_schema_description(
pa.schema(
[
pa.field("uniprot_id", pa.string(), nullable=False),
pa.field("db_reference", pa.string(), nullable=False),
pa.field("source", pa.string(), nullable=False),
]
),
"Standardised mapping data from UniProt",
)


Expand Down
10 changes: 10 additions & 0 deletions docs_macros/tables/pipeline_tables.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,12 @@
from cdm_data_loaders.converters.sources.json_schema.strict_scalars import NON_EXTENDABLE_SCALAR_HINTS
from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA
from cdm_data_loaders.pipelines.presets.pubmed import PubMedXmlToPyArrowSettings
from cdm_data_loaders.pipelines.presets.uniprot_cdm import (
UniProtSettings,
UniRef50CdmSettings,
UniRef90CdmSettings,
UniRef100CdmSettings,
)
from cdm_data_loaders.pipelines.presets.uniprot_xml import (
UniProtXmlToPyArrowSettings,
UniRef50XmlToPyArrowSettings,
Expand Down Expand Up @@ -37,9 +43,13 @@
XMLTOPYARROW_SETTINGS_CLASSES: Final[tuple[type[XmlToPyArrowSettings], ...]] = (
XmlToPyArrowSettings,
UniProtXmlToPyArrowSettings,
UniProtSettings,
UniRef50XmlToPyArrowSettings,
UniRef50CdmSettings,
UniRef90XmlToPyArrowSettings,
UniRef90CdmSettings,
UniRef100XmlToPyArrowSettings,
UniRef100CdmSettings,
PubMedXmlToPyArrowSettings,
)

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
title: UniProtKB CDM Fan-Out and Missing-Accession Rejection (`_rejected`)
pipeline: uniprot_cdm
settings_file: settings.toml
rules:
- R-XML-DEFAULT-NS
- R-XML-TRANSFORMS
- R-XML-REPORT
- R-UNIPROT-CDM-PRESET
notes: |
A `<uniprot>` XML file with one complete `<entry>` (`P12345`) and one `<entry>` whose `<accession>` is blank.
`UniProtSettings` fans the valid entry across `entity`, `identifier`, `name`, `protein`, and
`entity_x_publication` (`UNIPROT_CDM_SCHEMA_MAP`) and routes the blank-accession entry to `_rejected`
with `table_name="entry"` and `source_file="uniprot_sample.xml"`.
runs:
- id: default
input_dir: input
expected_report:
status: completed
ingested_files:
- uniprot_sample.xml
pipeline_load_info_errs: null
expected_counts:
entity: 1
identifier: 4
name: 3
protein: 1
entity_x_publication: 1
_rejected: 1
expected_tables: null
expected_json: null
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
<?xml version="1.0" encoding="UTF-8"?>
<uniprot xmlns="https://uniprot.org/uniprot">
<entry created="2000-05-30" modified="2020-01-01" version="3" dataset="Swiss-Prot">
<accession>P12345</accession>
<accession>Q67890</accession>
<name>TEST_ENTRY</name>
<protein>
<recommendedName>
<fullName>Test protein</fullName>
</recommendedName>
</protein>
<gene>
<name type="primary">testGene</name>
</gene>
<organism>
<name type="scientific">Test organism</name>
<dbReference type="NCBI Taxonomy" id="9606"/>
</organism>
<reference key="1">
<citation type="journal article" date="2020" name="Test Journal">
<title>A test paper.</title>
<authorList>
<person name="Tester T."/>
</authorList>
<dbReference type="PubMed" id="12345678"/>
</citation>
<scope>SEQUENCE</scope>
</reference>
<comment type="function">
<text>A test function.</text>
</comment>
<dbReference type="GeneID" id="1636"/>
<evidence type="ECO:0000255" key="1"/>
<proteinExistence type="evidence at protein level"/>
<sequence length="5" mass="500" checksum="ABCDEF0123456789" modified="2020-01-01" version="1">ACGTA</sequence>
</entry>
<entry created="2000-05-30" modified="2020-01-01" version="1" dataset="Swiss-Prot">
<accession> </accession>
<name>MISSING_ACC</name>
<protein>
<submittedName>
<fullName>Unindexed entry</fullName>
</submittedName>
</protein>
<organism>
<name type="scientific">Test organism</name>
<dbReference type="NCBI Taxonomy" id="9606"/>
</organism>
<proteinExistence type="uncertain"/>
<sequence length="4" mass="400" checksum="0123456789ABCDEF" modified="2020-01-01" version="1">ACGT</sequence>
</entry>
</uniprot>
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
loader_file_format = "parquet"
use_output_dir_for_pipeline_metadata = true
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
title: UniRef50 CDM Cluster Fan-Out and Missing-ID Rejection (`_rejected`)
pipeline: uniref50_cdm
settings_file: settings.toml
rules:
- R-XML-DEFAULT-NS
- R-XML-TRANSFORMS
- R-XML-REPORT
- R-UNIREF-CDM-PRESET
notes: |
A `<UniRef50>` XML file with one valid `<entry>` (`UniRef50_A0AB34IYJ6`) and one `<entry>` missing `id`.
`UniRef50CdmSettings` fans the valid cluster across `entity`, `cluster`, and `clustermember`
(`UNIREF_CDM_SCHEMA_MAP`) and routes the entry without `id` to `_rejected` with `table_name="entry"`
and `source_file="uniref50_sample.xml"`.
runs:
- id: default
input_dir: input
expected_report:
status: completed
ingested_files:
- uniref50_sample.xml
pipeline_load_info_errs: null
expected_counts:
entity: 1
cluster: 1
clustermember: 2
_rejected: 1
expected_tables: null
expected_json: null
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
<?xml version="1.0" encoding="UTF-8"?>
<UniRef50 xmlns="http://uniprot.org/uniref">
<entry id="UniRef50_A0AB34IYJ6" updated="2025-02-05">
<name>Cluster: PKZILLA-1 giant polyketide synthase</name>
<property type="member count" value="2"/>
<representativeMember>
<dbReference type="UniProtKB ID" id="A0AB34IYJ6_PRYPA">
<property type="UniProtKB accession" value="A0AB34IYJ6"/>
<property type="isSeed" value="true"/>
</dbReference>
<sequence length="4" checksum="385E596EB33D030E">ACGT</sequence>
</representativeMember>
<member>
<dbReference type="UniParc ID" id="UPI0001"/>
</member>
</entry>
<entry updated="2025-02-05">
<name>Cluster: Missing ID</name>
<representativeMember>
<dbReference type="UniParc ID" id="UPI0002"/>
</representativeMember>
</entry>
</UniRef50>
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
loader_file_format = "parquet"
use_output_dir_for_pipeline_metadata = true
31 changes: 26 additions & 5 deletions specs/pipelines/xmltopyarrow/rules.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ intro: |
1. **Target selection, preset configurations, and PyArrow schema resolution** (`XmlToPyArrowSettings`, `PresetXmlToPyArrowSettings`, `LevelPresetXmlToPyArrowSettings`).
2. **Streaming XML element extraction, default namespace resolution, and XSD-driven PyArrow table coercion** (`process_xml_file_to_pyarrow`).
3. **File discovery, `.gz` decompression, MD5 sidecar pre-processing, dead-letter routing (`_rejected`), table transforms, and pipeline reporting** (`run_xmltopyarrow_pipeline`).
4. **Preset subclass behavior** (`pubmed`, `uniprot`, `uniref50`, `uniref90`, `uniref100`).
4. **Preset subclass behavior** (`pubmed`, `uniprot`, `uniprot_cdm`, `uniref50`, `uniref50_cdm`, `uniref90`, `uniref90_cdm`, `uniref100`, `uniref100_cdm`).
5. **Errors** (`XmlToPyArrowConfigError` subclasses) raised for invalid target selection, invalid XSDs, or invalid pipeline hooks.

sections:
Expand Down Expand Up @@ -103,10 +103,11 @@ sections:
path), `line_no=None`, `raw_record=None`, and `error_detail`.
- id: R-XML-TRANSFORMS
statement: |
When `transforms_module` defines a `RecordTransformFn` for a target table, converted XML element dicts
are passed to the transform before buffering. The transform may drop records (`{}`), modify fields, or
fan out records across multiple destination tables, coercing each table with `OUTPUT_SCHEMAS` when
provided.
When `transforms_module` defines a `RecordTransformFn` for a target table, `route_and_coerce_record`
passes converted XML element dicts (and `TransformContext(table_name, source_file)` when the callable
declares a context parameter) to the transform before buffering. The transform may drop records (`{}`),
route rejected records to `_rejected` (`REJECTED_PYARROW_SCHEMA`) with `table_name` and `source_file`
provenance, update fields, or fan out records across destination tables coerced with `OUTPUT_SCHEMAS`.
- id: R-XML-REPORT
statement: |
Every pipeline execution writes one audit row to `_load_info` and a YAML summary to
Expand All @@ -132,3 +133,23 @@ sections:
When a `.xml.gz` file fails `validate_md5_sidecar` in `pubmed`, the file is omitted from
`ingested_files`, a file-level provenance row (`table_name=None`, `line_no=None`, `raw_record=None`)
is written to `_rejected`, and `report.txt` records `status: completed with errors` and `load_errors`.

- id: uniprot_cdm_presets
title: UniProtKB and UniRef CDM Preset Subclasses
rules:
- id: R-UNIPROT-CDM-PRESET
statement: |
`UniProtSettings` (`uniprot_cdm` CLI) subclasses `LockedTransformHooksMixin` and
`PresetXmlToPyArrowSettings`, locking `dataset_name="uniprot_cdm"`, `table_name="entry"`,
`xml_tag="entry"`, `transforms_module=None`, and `output_schemas_module=None`, and binding
`make_uniprot_cdm_transform` and `UNIPROT_CDM_SCHEMA_MAP` (`entity`, `identifier`, `name`,
`protein`, `entity_x_publication`). Entries missing `<accession>` or raising parse errors route
to `_rejected` with `table_name="entry"` and `source_file` provenance.
- id: R-UNIREF-CDM-PRESET
statement: |
`UniRef50CdmSettings`, `UniRef90CdmSettings`, and `UniRef100CdmSettings` (`uniref50_cdm`,
`uniref90_cdm`, `uniref100_cdm` CLIs) subclass `UniRefCdmSettings` (`LockedTransformHooksMixin`
and `PresetXmlToPyArrowSettings`), locking `dataset_name`, `table_name="entry"`, `xml_tag="entry"`,
`transforms_module=None`, and `output_schemas_module=None`, and binding `make_uniref_cdm_transform`
and `UNIREF_CDM_SCHEMA_MAP` (`entity`, `cluster`, `clustermember`). Entries missing `_id` or raising
parse errors route to `_rejected` with `table_name="entry"` and `source_file` provenance.
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,13 @@ rules:
notes: |
- `transforms_module="specs.pipelines.xmltopyarrow.transforms"` defines `TABLE_TRANSFORMS["measurements"] = transform_measurement` and `OUTPUT_SCHEMAS` for `measurements` and `measurement_tags`.
- `XmlToPyArrowSettings` skips default XSD-to-PyArrow schema generation for `measurements` and uses `OUTPUT_SCHEMAS`.
- `transform_measurement` drops `<measurement status="drop">` (`s2`), uppercases `_id` into `sensor_id`, computes `scaled_value`, and fans out `<tag>` values into `measurement_tags`.
- `transform_measurement` receives `TransformContext(table_name="measurements", source_file="data.xml")`, drops `<measurement status="drop">` (`s2`), routes `<measurement status="invalid">` (`s4`) to `_rejected` with `table_name` and `source_file` provenance, uppercases `_id` into `sensor_id`, computes `scaled_value`, and fans out `<tag>` values into `measurement_tags`.
runs:
- id: default
data: data.xml
expected_json: expected.json
expected_table: expected_table.py
expected_rejected: expected_rejected.json
extra_expected_json:
measurement_tags: expected_tags.json
params:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,8 @@
<measurement id="s3" status="ok">
<value>3.5</value>
</measurement>
<measurement id="s4" status="invalid">
<value>-1.0</value>
</measurement>
</report>

Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
[
{
"error_detail": "Invalid measurement status",
"line_no": null,
"raw_record": "{\"_id\": \"s4\", \"_status\": \"invalid\", \"value\": -1.0}",
"source_file": "data.xml",
"table_name": "measurements"
}
]
14 changes: 11 additions & 3 deletions specs/pipelines/xmltopyarrow/transforms.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,9 @@

import pyarrow as pa

from cdm_data_loaders.core.buffer import make_rejected_table_result
from cdm_data_loaders.core.md5_preprocessor import validate_md5_sidecar
from cdm_data_loaders.core.registry import FilePreprocessFn, RecordTransformFn
from cdm_data_loaders.core.registry import FilePreprocessFn, RecordTransformFn, TransformContext

FILE_PREPROCESSORS: tuple[FilePreprocessFn, ...] = (validate_md5_sidecar,)

Expand All @@ -25,10 +26,17 @@
)


def transform_measurement(row: dict[str, Any]) -> dict[str, list[dict[str, Any]]]:
"""Drop draft measurements, compute ``scaled_value``, and fan out tags to ``measurement_tags``."""
def transform_measurement(row: dict[str, Any], ctx: TransformContext) -> dict[str, list[dict[str, Any]]]:
"""Drop draft rows, reject invalid rows with ``ctx`` provenance, compute ``scaled_value``, and fan out tags."""
if row.get("_status") == "drop":
return {}
if row.get("_status") == "invalid":
return make_rejected_table_result(
ctx.table_name,
row,
"Invalid measurement status",
source_file=ctx.source_file,
)
sensor_id = str(row["_id"]).upper()
value = float(row["value"])
routed: dict[str, list[dict[str, Any]]] = {
Expand Down
2 changes: 1 addition & 1 deletion src/cdm_data_loaders/cli/ncbi_ftp_filter_updates.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@

from cdm_data_loaders.core.cli import CliRunner
from cdm_data_loaders.core.settings import LoggerSettings, default_settings_with_shortcuts
from cdm_data_loaders.datasets.ncbi_ftp.constants import Source

Check warning on line 13 in src/cdm_data_loaders/cli/ncbi_ftp_filter_updates.py

View check run for this annotation

Codecov / codecov/patch

src/cdm_data_loaders/cli/ncbi_ftp_filter_updates.py#L13

Added line #L13 was not covered by tests
from cdm_data_loaders.datasets.ncbi_ftp.updates import (
BATCH_SIZE,
Source,
Splits,
build_splits,
compute_batch_width,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,6 @@ def _reader(self) -> StrictJsonSchemaReader:
:rtype: StrictJsonSchemaReader
"""
forbidden_formats = frozenset(fmt for fmt, dtype in self.format_map.items() if is_non_json_arrow_type(dtype))
forbidden_formats = frozenset(fmt for fmt, dtype in self.format_map.items() if is_non_json_arrow_type(dtype))
return StrictJsonSchemaReader(
self.extension_registry,
error_type=self._error_cls,
Expand Down
Loading
Loading