diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ae048e7f..62bd9cc0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -47,15 +47,21 @@ jobs: strategy: fail-fast: false matrix: - suite: + suite: &basic_suites - name: pipelines args: tests/cdm_data_loaders/pipelines/ - name: e2e - args: tests/e2e/ + markers: "not requires_ceph" + args: --ignore=tests/e2e/ncbi_ftp/ --ignore=tests/e2e/reference/ tests/e2e/ + - name: ncbi_ftp_e2e + args: tests/e2e/ncbi_ftp/ + - name: reference_e2e + args: tests/e2e/reference/ - name: io args: tests/cdm_data_loaders/io/ - name: everything else - args: --ignore=tests/e2e --ignore=tests/cdm_data_loaders/pipelines --ignore=tests/docs --ignore=tests/cdm_data_loaders/io + markers: "not requires_spark" + args: --ignore=tests/e2e/ --ignore=tests/cdm_data_loaders/pipelines/ --ignore=tests/docs/ --ignore=tests/cdm_data_loaders/io/ steps: - name: Checkout @@ -93,28 +99,9 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.13", "3.14"] + python-version: ["3.13", "3.14", "3.15"] os: ["ubuntu-latest"] - suite: - - name: pipelines - markers: "not requires_spark and not requires_ceph and not docs and not state_audit" - args: tests/cdm_data_loaders/pipelines/ - - name: e2e - markers: "not requires_spark and not requires_ceph and not docs and not state_audit" - args: tests/e2e/ - - name: io - markers: "not requires_spark and not requires_ceph and not docs and not state_audit" - args: tests/cdm_data_loaders/io/ - - name: everything else - markers: "not requires_spark and not requires_ceph and not docs and not state_audit" - args: --ignore=tests/e2e --ignore=tests/cdm_data_loaders/pipelines --ignore=tests/docs --ignore=tests/cdm_data_loaders/io - include: - - python-version: "3.13" - os: "ubuntu-latest" - suite: - name: state_audit - markers: "state_audit" - args: --no-cov tests/harness/test_global_state_audit.py + suite: *basic_suites steps: - name: Checkout @@ -180,3 +167,51 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} report_type: test_results + + state_audit: + name: Run state audit (${{ matrix.suite.name }}) + runs-on: ubuntu-latest + needs: code_format_and_lint + strategy: + fail-fast: false + matrix: + suite: *basic_suites + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 + + - name: Install uv + uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 + with: + enable-cache: true + + - name: Set up Python + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 + with: + python-version: "3.14" + + - name: Install dependencies + run: uv sync --all-groups + + - name: Install qsv + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + QSV_VERSION=$(gh release view --repo dathere/qsv --json tagName --jq .tagName) + ASSET="qsv-${QSV_VERSION#v}-x86_64-unknown-linux-gnu.zip" + gh release download "${QSV_VERSION}" --repo dathere/qsv --pattern "${ASSET}" --dir "${RUNNER_TEMP}" --clobber + unzip -q -o "${RUNNER_TEMP}/${ASSET}" qsv -d "${RUNNER_TEMP}/qsv" + sudo install -m 0755 "${RUNNER_TEMP}/qsv/qsv" /usr/local/bin/qsv + rm -rf "${RUNNER_TEMP}/qsv" "${RUNNER_TEMP}/${ASSET}" + qsv --version + + - name: Run state audit + shell: bash + continue-on-error: false + run: > + uv run --all-groups pytest --state-audit -qq --tb=short -p no:randomly -p no:cov + -m "not requires_spark and not requires_ceph and not external_request and not docker and not docs and not state_audit" + --ignore=tests/harness/test_global_state_audit.py + ${{ matrix.suite.args }} diff --git a/src/cdm_data_loaders/core/filesystem.py b/src/cdm_data_loaders/core/filesystem.py index 06ec191e..ca48a538 100644 --- a/src/cdm_data_loaders/core/filesystem.py +++ b/src/cdm_data_loaders/core/filesystem.py @@ -99,14 +99,14 @@ def _collect_relative_paths(fs_resources: Iterable[DltResource]) -> list[str]: :return: sorted unique POSIX relative file paths :rtype: list[str] """ - paths: set[str] = set() - for fs_resource in fs_resources: - for yielded in fs_resource: - batch = [yielded] if isinstance(yielded, dict) else yielded - for item in batch: - if isinstance(item, dict) and "relative_path" in item: - paths.add(str(item["relative_path"])) - return sorted(paths) + return sorted( + { + str(item["relative_path"]) + for fs_resource in fs_resources + for item in fs_resource + if isinstance(item, dict) and "relative_path" in item + } + ) def list_input_dir_files(fs_resource: DltResource, input_dir: str) -> list[str]: diff --git a/tests/cdm_data_loaders/core/test_filesystem.py b/tests/cdm_data_loaders/core/test_filesystem.py index ae40c48d..c94ebccd 100644 --- a/tests/cdm_data_loaders/core/test_filesystem.py +++ b/tests/cdm_data_loaders/core/test_filesystem.py @@ -16,6 +16,7 @@ from cdm_data_loaders.core.buffer import REJECTED_PYARROW_SCHEMA from cdm_data_loaders.core.filesystem import ( FilesystemTracker, + _collect_relative_paths, attach_file_collector, build_file_ingest_kwargs, build_filesystem_resource, @@ -128,6 +129,41 @@ def trans_a2(items: TDataItems) -> Iterator[TDataItems]: assert [[item["relative_path"] for item in res] for res in found] == expected_paths +@pytest.mark.parametrize( + ("resource_batches", "expected"), + [ + pytest.param([], [], id="empty_resources"), + pytest.param( + [ + [{"relative_path": "z.txt"}, {"relative_path": "a.txt"}], + [{"relative_path": "a.txt"}, {"relative_path": "m.txt"}], + ], + ["a.txt", "m.txt", "z.txt"], + id="deduplicates_and_sorts_across_resources", + ), + pytest.param( + [ + [ + "plain_string", + 42, + {"file_url": "file:///tmp/ignored.xml"}, + {"relative_path": "kept.xml"}, + ] + ], + ["kept.xml"], + id="ignores_non_dicts_and_missing_relative_path", + ), + ], +) +def test_collect_relative_paths_pass( + resource_batches: list[list[Any]], + expected: list[str], +) -> None: + """_collect_relative_paths returns sorted unique relative_path values and ignores non-matching items.""" + resources = [dlt.resource(batch, name=f"fs_{idx}") for idx, batch in enumerate(resource_batches)] + assert _collect_relative_paths(resources) == expected + + @pytest.mark.parametrize( ("target_kind", "expected"), [