Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 59 additions & 24 deletions .github/workflows/tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,15 +47,21 @@ jobs:
strategy:
fail-fast: false
matrix:
suite:
suite: &basic_suites
- name: pipelines
args: tests/cdm_data_loaders/pipelines/
- name: e2e
args: tests/e2e/
markers: "not requires_ceph"
args: --ignore=tests/e2e/ncbi_ftp/ --ignore=tests/e2e/reference/ tests/e2e/
- name: ncbi_ftp_e2e
args: tests/e2e/ncbi_ftp/
- name: reference_e2e
args: tests/e2e/reference/
- name: io
args: tests/cdm_data_loaders/io/
- name: everything else
args: --ignore=tests/e2e --ignore=tests/cdm_data_loaders/pipelines --ignore=tests/docs --ignore=tests/cdm_data_loaders/io
markers: "not requires_spark"
args: --ignore=tests/e2e/ --ignore=tests/cdm_data_loaders/pipelines/ --ignore=tests/docs/ --ignore=tests/cdm_data_loaders/io/

steps:
- name: Checkout
Expand Down Expand Up @@ -93,28 +99,9 @@ jobs:
strategy:
fail-fast: false
matrix:
python-version: ["3.13", "3.14"]
python-version: ["3.13", "3.14", "3.15"]
os: ["ubuntu-latest"]
suite:
- name: pipelines
markers: "not requires_spark and not requires_ceph and not docs and not state_audit"
args: tests/cdm_data_loaders/pipelines/
- name: e2e
markers: "not requires_spark and not requires_ceph and not docs and not state_audit"
args: tests/e2e/
- name: io
markers: "not requires_spark and not requires_ceph and not docs and not state_audit"
args: tests/cdm_data_loaders/io/
- name: everything else
markers: "not requires_spark and not requires_ceph and not docs and not state_audit"
args: --ignore=tests/e2e --ignore=tests/cdm_data_loaders/pipelines --ignore=tests/docs --ignore=tests/cdm_data_loaders/io
include:
- python-version: "3.13"
os: "ubuntu-latest"
suite:
name: state_audit
markers: "state_audit"
args: --no-cov tests/harness/test_global_state_audit.py
suite: *basic_suites

steps:
- name: Checkout
Expand Down Expand Up @@ -180,3 +167,51 @@ jobs:
with:
token: ${{ secrets.CODECOV_TOKEN }}
report_type: test_results

state_audit:
name: Run state audit (${{ matrix.suite.name }})
runs-on: ubuntu-latest
needs: code_format_and_lint
strategy:
fail-fast: false
matrix:
suite: *basic_suites

steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1

- name: Install uv
uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2
with:
enable-cache: true

- name: Set up Python
uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0
with:
python-version: "3.14"

- name: Install dependencies
run: uv sync --all-groups

- name: Install qsv
shell: bash
env:
GH_TOKEN: ${{ github.token }}
run: |
QSV_VERSION=$(gh release view --repo dathere/qsv --json tagName --jq .tagName)
ASSET="qsv-${QSV_VERSION#v}-x86_64-unknown-linux-gnu.zip"
gh release download "${QSV_VERSION}" --repo dathere/qsv --pattern "${ASSET}" --dir "${RUNNER_TEMP}" --clobber
unzip -q -o "${RUNNER_TEMP}/${ASSET}" qsv -d "${RUNNER_TEMP}/qsv"
sudo install -m 0755 "${RUNNER_TEMP}/qsv/qsv" /usr/local/bin/qsv
rm -rf "${RUNNER_TEMP}/qsv" "${RUNNER_TEMP}/${ASSET}"
qsv --version

- name: Run state audit
shell: bash
continue-on-error: false
run: >
uv run --all-groups pytest --state-audit -qq --tb=short -p no:randomly -p no:cov
-m "not requires_spark and not requires_ceph and not external_request and not docker and not docs and not state_audit"
--ignore=tests/harness/test_global_state_audit.py
${{ matrix.suite.args }}
16 changes: 8 additions & 8 deletions src/cdm_data_loaders/core/filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,14 +99,14 @@
:return: sorted unique POSIX relative file paths
:rtype: list[str]
"""
paths: set[str] = set()
for fs_resource in fs_resources:
for yielded in fs_resource:
batch = [yielded] if isinstance(yielded, dict) else yielded
for item in batch:
if isinstance(item, dict) and "relative_path" in item:
paths.add(str(item["relative_path"]))
return sorted(paths)
return sorted(

Check warning on line 102 in src/cdm_data_loaders/core/filesystem.py

View check run for this annotation

Codecov / codecov/patch

src/cdm_data_loaders/core/filesystem.py#L102

Added line #L102 was not covered by tests
{
str(item["relative_path"])
for fs_resource in fs_resources
for item in fs_resource
if isinstance(item, dict) and "relative_path" in item
}
)


def list_input_dir_files(fs_resource: DltResource, input_dir: str) -> list[str]:
Expand Down
36 changes: 36 additions & 0 deletions tests/cdm_data_loaders/core/test_filesystem.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA
from cdm_data_loaders.core.filesystem import (
FilesystemTracker,
_collect_relative_paths,
attach_file_collector,
build_file_ingest_kwargs,
build_filesystem_resource,
Expand Down Expand Up @@ -128,6 +129,41 @@ def trans_a2(items: TDataItems) -> Iterator[TDataItems]:
assert [[item["relative_path"] for item in res] for res in found] == expected_paths


@pytest.mark.parametrize(
("resource_batches", "expected"),
[
pytest.param([], [], id="empty_resources"),
pytest.param(
[
[{"relative_path": "z.txt"}, {"relative_path": "a.txt"}],
[{"relative_path": "a.txt"}, {"relative_path": "m.txt"}],
],
["a.txt", "m.txt", "z.txt"],
id="deduplicates_and_sorts_across_resources",
),
pytest.param(
[
[
"plain_string",
42,
{"file_url": "file:///tmp/ignored.xml"},
{"relative_path": "kept.xml"},
]
],
["kept.xml"],
id="ignores_non_dicts_and_missing_relative_path",
),
],
)
def test_collect_relative_paths_pass(
resource_batches: list[list[Any]],
expected: list[str],
) -> None:
"""_collect_relative_paths returns sorted unique relative_path values and ignores non-matching items."""
resources = [dlt.resource(batch, name=f"fs_{idx}") for idx, batch in enumerate(resource_batches)]
assert _collect_relative_paths(resources) == expected


@pytest.mark.parametrize(
("target_kind", "expected"),
[
Expand Down
Loading