Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
not_for_github/
cdl_skill/
mutants/
.DS_Store
mutants/

Expand Down
2 changes: 2 additions & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ models = [
]
mutants = [
"mutmut>=3.8.1",
"mutmut-mcp",
]
docs = [
"griffe-pydantic>=1.3.2",
Expand Down Expand Up @@ -287,4 +288,5 @@ DLT_PROJECT_DIR = "/tmp/dlt"
[tool.uv.sources]
cdm-schema = { git = "https://github.com/kbase/cdm-schema.git" }
cdm-task-service-client = { git = "https://github.com/kbase/cdm-task-service-client.git" }
mutmut-mcp = { git = "https://github.com/wdm0006/mutmut-mcp" }
# berdl-notebook-utils = { git = "https://github.com/KBaseDataLakehouse/spark_notebook.git", subdirectory = "notebook_utils" }
11 changes: 11 additions & 0 deletions tests/cdm_data_loaders/cli/test_gz.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,17 @@ def test_cli_directory_with_glob(tmp_path: Path, source_flag: str, glob_flag: st
assert not (sub_dir / "b.log.gz").exists()


def test_main_directory_default_glob(tmp_path: Path) -> None:
"""Compress all files in a directory using ``*`` when ``--file-glob`` is omitted."""
sub_dir = tmp_path / "sub"
sub_dir.mkdir()
for name in ("a.txt", "b.log"):
(sub_dir / name).write_text(name, encoding="utf-8")
main(["--source", str(sub_dir)])
assert (sub_dir / "a.txt.gz").exists()
assert (sub_dir / "b.log.gz").exists()


def test_cli_main_nonexistent_source(tmp_path: Path) -> None:
"""Raise ``RuntimeError`` when ``--source`` points to a missing path."""
missing = tmp_path / "doesnotexist"
Expand Down
39 changes: 18 additions & 21 deletions tests/cdm_data_loaders/cli/test_ncbi_ftp_promote.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,12 +102,6 @@
"dry_run": False,
}

MOCK_REPORT_WITH_FAILURES: Final[dict[str, Any]] = {
**MOCK_REPORT_SUCCESS,
"promoted": 3,
"failed": 2,
}


def make_settings(**kwargs: str | int | bool | Path | PurePosixPath | None) -> PromoteSettings:
"""Build a validated ``PromoteSettings`` instance with a default ``staging_path``.
Expand Down Expand Up @@ -212,28 +206,31 @@ def test_run_promote_calls_promote_from_s3_with_correct_args(tmp_path: Path) ->
)


def test_run_promote_no_error_on_zero_failures() -> None:
"""run_promote does not raise when promote_from_s3 reports zero failures."""
config = make_settings()
with patch(
"cdm_data_loaders.cli.ncbi_ftp_promote.promote_from_s3",
return_value=MOCK_REPORT_SUCCESS,
):
run_promote(config)


def test_run_promote_logs_error_on_failures(caplog: pytest.LogCaptureFixture) -> None:
"""run_promote logs an error without raising when promote_from_s3 reports failures."""
@pytest.mark.parametrize(
("failed_count", "expected_messages"),
[
pytest.param(0, [], id="zero-failures"),
pytest.param(1, ["Promote completed with 1 failures"], id="single-failure"),
pytest.param(2, ["Promote completed with 2 failures"], id="multiple-failures"),
],
)
def test_run_promote_logs_error_only_when_failures_positive(
caplog: pytest.LogCaptureFixture,
failed_count: int,
expected_messages: list[str],
) -> None:
"""run_promote logs an error without raising if and only if promote_from_s3 reports failures > 0."""
config = make_settings()
report = {**MOCK_REPORT_SUCCESS, "failed": failed_count}
with (
patch(
"cdm_data_loaders.cli.ncbi_ftp_promote.promote_from_s3",
return_value=MOCK_REPORT_WITH_FAILURES,
return_value=report,
),
caplog.at_level(logging.ERROR),
caplog.at_level(logging.ERROR, logger="cdm_data_loaders.cli.ncbi_ftp_promote"),
):
run_promote(config)
assert "Promote completed with 2 failures" in caplog.text
assert caplog.messages == expected_messages


def test_run_promote_dry_run_forwarded() -> None:
Expand Down
46 changes: 46 additions & 0 deletions tests/cdm_data_loaders/cli/test_ncbi_ftp_sync.py
Original file line number Diff line number Diff line change
@@ -1,13 +1,16 @@
"""Tests for cli.ncbi_ftp_sync — settings and CLI shortcuts."""

import logging
from pathlib import Path
from unittest.mock import patch

import pytest
from pydantic import ValidationError
from pydantic_settings import CliApp

from cdm_data_loaders.cli.ncbi_ftp_sync import (
DownloadSettings,
run_download,
)
from cdm_data_loaders.core.fields import INPUT_MOUNT, OUTPUT_MOUNT
from cdm_data_loaders.datasets.ncbi_ftp.constants import FTP_HOST
Expand Down Expand Up @@ -105,3 +108,46 @@ def test_settings_threads_boundaries_accepted(threads: int) -> None:
"""Boundary thread counts (1 and 32) are accepted."""
s = DownloadSettings(threads=threads) # pyright: ignore[reportCallIssue]
assert s.threads == threads


@pytest.mark.parametrize(
("failed_count", "expected_messages"),
[
pytest.param(0, [], id="no-failures"),
pytest.param(1, ["Download completed with 1 failures"], id="single-failure"),
pytest.param(2, ["Download completed with 2 failures"], id="multiple-failures"),
],
)
def test_run_download_forwards_settings_and_logs_failures(
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
failed_count: int,
expected_messages: list[str],
) -> None:
"""run_download forwards all DownloadSettings fields to download_batch and logs failures > 0."""
manifest = tmp_path / "manifest.txt"
out_dir = tmp_path / "out"
settings = DownloadSettings(
manifest=manifest,
output_dir=out_dir,
threads=CUSTOM_THREADS,
ftp_host="ftp.example.org",
limit=CUSTOM_LIMIT,
)
with (
patch(
"cdm_data_loaders.cli.ncbi_ftp_sync.download_batch",
return_value={"failed": failed_count},
) as mock_batch,
caplog.at_level(logging.ERROR, logger="cdm_data_loaders.cli.ncbi_ftp_sync"),
):
run_download(settings)

mock_batch.assert_called_once_with(
manifest_path=manifest,
output_dir=out_dir,
threads=CUSTOM_THREADS,
ftp_host="ftp.example.org",
limit=CUSTOM_LIMIT,
)
assert caplog.messages == expected_messages
120 changes: 120 additions & 0 deletions tests/cdm_data_loaders/cli/test_polaris_schema_definitions.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
"""Unit and integration tests for cdm_data_loaders.cli.polaris_schema_definitions."""

import json
import logging
from collections.abc import Generator
from contextlib import contextmanager
from pathlib import Path
from typing import Any, Final
from unittest.mock import patch

import dlt
import pyarrow as pa
Expand All @@ -16,7 +19,9 @@
from pyiceberg.utils.config import Config

from cdm_data_loaders.cli.polaris_schema_definitions import (
PolarisSchemaDefinitionsSettings,
load_pyarrow_schema,
run_from_settings,
)
from cdm_data_loaders.cli.polaris_schema_definitions import (
main as cli_main,
Expand Down Expand Up @@ -722,3 +727,118 @@ def test_cli_main_applies_definitions_and_prints_json_report(
updated_tbl = verify_cat.load_table((NAMESPACE, "id_mapping"))
assert updated_tbl.properties["comment"] == "Standardised mapping data from UniProt"
assert collect_iceberg_docs(updated_tbl.schema()) == parsed_stdout["updated_fields"]


def test_load_pyarrow_schema_blank_description_returns_none(tmp_path: Path) -> None:
"""load_pyarrow_schema produces no metadata definition for whitespace-only descriptions."""
json_path = tmp_path / "blank_desc.json"
json_path.write_text(
json.dumps(
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"description": " ",
"properties": {"id": {"type": "string"}},
}
),
encoding="utf-8",
)
loaded = load_pyarrow_schema(str(json_path))
assert extract_metadata_definition(loaded.metadata) is None


@pytest.mark.parametrize(
("dry_run", "expected_verb"),
[
pytest.param(True, "Planned", id="dry-run"),
pytest.param(False, "Applied", id="apply"),
],
)
def test_run_from_settings_forwards_all_options_and_logs(
tmp_path: Path,
caplog: pytest.LogCaptureFixture,
*,
dry_run: bool,
expected_verb: str,
) -> None:
"""run_from_settings forwards catalog and sync options and logs Planned or Applied."""
json_path = tmp_path / "schema.json"
json_path.write_text(
json.dumps(
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"properties": {"id": {"type": "string"}},
}
),
encoding="utf-8",
)

captured_cfg: list[PolarisCatalogConfig] = []
captured_opts: list[DefinitionSyncOptions] = []

@contextmanager
def fake_connect(cfg: PolarisCatalogConfig) -> Generator[object]:
captured_cfg.append(cfg)
yield object()

def fake_apply(
_cat: object,
_table: str,
_schema: pa.Schema,
*,
options: DefinitionSyncOptions,
) -> SchemaDefinitionsReport:
captured_opts.append(options)
return SchemaDefinitionsReport(
table_identifier=("ns", "tbl"),
dry_run=options.dry_run,
table_comment_applied="desc",
updated_fields={"a": "b"},
unchanged_fields=["c"],
)

bearer_val = "tok"
settings = PolarisSchemaDefinitionsSettings(
catalog="mycat",
table="ns.tbl",
schema_source=str(json_path),
table_description="override desc",
uri="http://polaris:8181/api/catalog",
warehouse="wh",
credential="cid:csec",
token=bearer_val,
scope="ROLE:ALL",
metadata_keys=("doc",),
normalize_root_names=False,
strict=True,
dry_run=dry_run,
)
with (
patch("cdm_data_loaders.cli.polaris_schema_definitions.connect_polaris_catalog", fake_connect),
patch("cdm_data_loaders.cli.polaris_schema_definitions.apply_schema_definitions", fake_apply),
caplog.at_level(logging.INFO, logger="cdm_data_loaders.cli.polaris_schema_definitions"),
):
report = run_from_settings(settings)

assert report.dry_run is dry_run
assert captured_cfg == [
PolarisCatalogConfig(
catalog="mycat",
uri="http://polaris:8181/api/catalog",
warehouse="wh",
credential="cid:csec",
token=bearer_val,
scope="ROLE:ALL",
)
]
assert captured_opts == [
DefinitionSyncOptions(
table_description="override desc",
metadata_keys=("doc",),
normalize_root_names=False,
strict=True,
dry_run=dry_run,
)
]
assert f"{expected_verb} definitions on ns.tbl" in caplog.text
Loading
Loading