From 7484f3f0d92b2e9d241df21858a78b0a05eccca9 Mon Sep 17 00:00:00 2001 From: Jody Klymak Date: Fri, 3 Jul 2026 15:40:03 -0700 Subject: [PATCH 1/8] .gitignore add pixi --- .github/workflows/tests.yml | 64 +++++++++++++++++++++++++++++++++++++ .gitignore | 10 ++++++ conftest.py | 5 +++ 3 files changed, 79 insertions(+) create mode 100644 .github/workflows/tests.yml create mode 100644 conftest.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..445dd28 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,64 @@ +name: Tests + +on: + push: + branches: [master] + pull_request: + +jobs: + test: + name: ${{ matrix.os }} / py${{ matrix.python-version }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + python-version: ["3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: uv sync --group dev + + # setuptools isn't a runtime/dev dependency, only needed transiently + # to drive build_ext here. + - name: Build C extension in place + run: uv run --with setuptools python setup.py build_ext --inplace + + - name: Confirm C extension is importable + shell: bash + run: uv run python -c "import _dbdreader; print('C extension:', _dbdreader.__file__)" + + - name: Run tests (C extension) + shell: bash + env: + DBDREADER_C_EXTENSION: "1" + run: uv run pytest tests -v + + - name: Run tests (pure-Python fallback) + shell: bash + env: + DBDREADER_C_EXTENSION: "0" + run: uv run pytest tests -v + + docs: + name: Build docs + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Install dependencies + run: uv sync --group dev + + - name: Build Sphinx docs + run: uv run sphinx-build -W -b html docs/source docs/build/html diff --git a/.gitignore b/.gitignore index cf786f0..799f82e 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,13 @@ MANIFEST *.so # this cac file gets created anyway. data/cac/daad1b20.cac +uv.lock +dbdreader/data/01600000.dbd +dbdreader/data/02380108.ebd + +# pixi things in case user puts those in: +pixi.lock +pixi.toml +.pixi/ + +.gitattributes diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..0a8d3f3 --- /dev/null +++ b/conftest.py @@ -0,0 +1,5 @@ +import sys +import os + +# Ensure tests always import from the local source tree, not any installed package. +sys.path.insert(0, os.path.dirname(__file__)) From f22acf221cf7707bbb7ddf7e2edbf15e7a81f5fd Mon Sep 17 00:00:00 2001 From: Jody Klymak Date: Fri, 3 Jul 2026 16:11:28 -0700 Subject: [PATCH 2/8] bump python to 3.11 --- .github/workflows/pypi-publish.yml | 2 +- .github/workflows/test-publish.yml | 2 +- .github/workflows/tests.yml | 2 +- pyproject.toml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 8cecbdc..712c05d 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -20,7 +20,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 env: - CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-*" + CIBW_BUILD: "cp311-* cp312-* cp313-*" CIBW_SKIP: "*-win32 *_i686 *musllinux*" - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml index f6706c9..e893725 100644 --- a/.github/workflows/test-publish.yml +++ b/.github/workflows/test-publish.yml @@ -17,7 +17,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 env: - CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-*" + CIBW_BUILD: "cp311-* cp312-* cp313-*" CIBW_SKIP: "*-win32 *_i686 *musllinux*" - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 445dd28..cd8bc40 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.10", "3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index 054c973..c7e4223 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "dbdreader" version = "0.6.1" description = "A python module to access binary data files generated by Teledyne WebbResearch gliders" readme = "README.md" -requires-python = ">=3.10" +requires-python = ">=3.11" authors = [ {name = "Lucas Merckelbach", email = "lucas.merckelbach@hereon.de"} ] From 4daf9470400ef32282040f0b8efc8d35218d5174 Mon Sep 17 00:00:00 2001 From: Jody Klymak Date: Fri, 3 Jul 2026 16:51:13 -0700 Subject: [PATCH 3/8] FIX: fix decompression macOS --- extension/decompress.c | 18 +++++++++---- tests/test_decompress.py | 55 +++++++++++++++++++++++++--------------- 2 files changed, 48 insertions(+), 25 deletions(-) diff --git a/extension/decompress.c b/extension/decompress.c index eca9a0a..ef773bc 100644 --- a/extension/decompress.c +++ b/extension/decompress.c @@ -90,15 +90,23 @@ static FILE* fopen_compressed_file_fopen(const char* filename, int* errorno){ fpmem = fopen(base, "rb"); } else{ + /* Decompression failed partway through, so the base file on + disk is incomplete/corrupt. Remove it -- otherwise a later + call would find this partial file, mistake it for a + complete, previously-decompressed file, and silently return + truncated data instead of re-reporting the error. */ + remove(base); fpmem=NULL; } } + else { + /* Could not create the base file for writing. This is fatal. */ + *errorno = ERROR_FAILED_TO_WRITE_BASE_FILE; + } } - else { - /* Writing operation failed. This is fatal. */ - *errorno = ERROR_FAILED_TO_WRITE_BASE_FILE; - fpmem=NULL; - } + /* else: the decompressed file already exists on disk (written by + a previous call), so just use it as-is -- fpmem is already open + for reading and *errorno is NO_ERROR. */ /* Here fpmem is either NULL (and all failed, or it points to the decompressed file.*/ free(extension_decompressed); diff --git a/tests/test_decompress.py b/tests/test_decompress.py index f20765c..243c56e 100644 --- a/tests/test_decompress.py +++ b/tests/test_decompress.py @@ -1,6 +1,5 @@ import os from hashlib import md5 -from itertools import chain if not __name__ == '__main__': from pytest import fixture @@ -23,8 +22,8 @@ def load_verification_data(): with open(filename_verify, 'r') as fp: data = fp.read() return data - -# Open a file and lazy read block by block. Compares with + +# Open a file and lazy read block by block. Compares with # uncompressed file # def test_read_file(load_verification_data): @@ -36,7 +35,7 @@ def test_read_file(load_verification_data): data += block assert data.decode('ascii') == verification_data -# Open a file and decompress whole file. Compares with +# Open a file and decompress whole file. Compares with # uncompressed file # def test_read_file_in_memory(load_verification_data): @@ -46,8 +45,8 @@ def test_read_file_in_memory(load_verification_data): data = d.decompress() assert data.decode('ascii') == verification_data -# Open a file and decompress first block only. -# +# Open a file and decompress first block only. +# # def test_read_file_one_block_only(): filename = 'dbdreader/data/01600000.mcg' @@ -55,7 +54,7 @@ def test_read_file_one_block_only(): blocks = [block for block in d.decompressed_blocks(n=1)] assert len(blocks) == 1 -# Open a file and lazy read block by block. Compares with +# Open a file and lazy read block by block. Compares with # uncompressed file. Instead of using with statement, open file # explicitly def test_read_file_explicit_file_opener(load_verification_data): @@ -68,7 +67,7 @@ def test_read_file_explicit_file_opener(load_verification_data): data += block assert data.decode('ascii') == verification_data -# Open a file and decompress whole file. Compares with +# Open a file and decompress whole file. Compares with # uncompressed file. Instead of using with statement, open file # explicitly def test_read_file_in_memory_explicit_file_opener(load_verification_data): @@ -130,8 +129,8 @@ def test_extension_generator_with_invalid_extension(): s = fd._generate_filename_for_output('01600000.cd') - -# Open a file and lazy read block by block. Check cac filesCompares with + +# Open a file and lazy read block by block. Check cac filesCompares with # uncompressed file # def test_read_ccc_file(): @@ -167,8 +166,8 @@ def test_CompressedFileReadlines(load_verification_data): data = b"".join(lines) assert data.decode('ascii') == verification_data - - + + # Test reading of data/01600001.dcd and check it is identical to data/01600001.dbd def test_read_compressed_file_C_code(): compressed_filename = 'dbdreader/data/01600001.dcd' @@ -178,7 +177,7 @@ def test_read_compressed_file_C_code(): compressed_data = compressed_dbd.get("m_depth") regular_data = regular_dbd.get("m_depth") assert np.all(compressed_data[0]==regular_data[0]) and np.all(compressed_data[1]==regular_data[1]) - + # Test reading of data/0160000?.dcd MultiDBD def test_read_compressed_files_C_code(): pattern = 'dbdreader/data/0160000?.dcd' @@ -193,7 +192,7 @@ def test_missing_cac_file_for_single_compressed_file(): os.unlink('dbdreader/data/cac/daad1b20.cac') except: pass - + with pytest.raises(dbdreader.DbdError): dbd = dbdreader.DBD('dbdreader/data/01600000.ecd', cacheDir='dbdreader/data/cac') @@ -203,7 +202,7 @@ def test_missing_cac_file_for_compressed_file_multidbd(): os.unlink('dbdreader/data/cac/daad1b20.cac') except: pass - + dbd = dbdreader.MultiDBD('dbdreader/data/0160000?.ecd', cacheDir='dbdreader/data/cac') assert os.path.exists('dbdreader/data/cac/daad1b20.cac') @@ -227,7 +226,23 @@ def test_handle_corrupt_compressed_file_multidbd(): data_both = dbd.get("sci_water_temp", "sci_water_cond", continue_on_reading_error=True) dbd = dbdreader.DBD('dbdreader/data/02380107.ecd', cacheDir='dbdreader/data/cac') data_one = dbd.get("sci_water_temp", "sci_water_cond") - assert np.all([np.all(x==y) for x, y in zip(chain(*data_both), chain(*data_one))]) + labels = [ + "time[sci_water_temp]", + "sci_water_temp", + "time[sci_water_cond]", + "sci_water_cond", + ] + assert len(data_both) == len(data_one), ( + f"Returned tuple length differs: {len(data_both)} vs {len(data_one)}" + ) + + for idx, (arr_both, arr_one) in enumerate(zip(data_both, data_one)): + label = labels[idx] if idx < len(labels) else f"output[{idx}]" + np.testing.assert_array_equal( + arr_both, + arr_one, + err_msg=f"Mismatch in {label}", + ) # Test handling of files that fail to decompress properly @@ -236,7 +251,7 @@ def test_handle_failing_decompression_multidbd(): t, T = dbd.get("sci_water_temp") # we should get 126 values assert len(t) == 126 - + # Test handling of files that fail to decompress properly def test_handle_failing_decompression_dbd(): @@ -244,7 +259,7 @@ def test_handle_failing_decompression_dbd(): with pytest.raises(dbdreader.DbdError) as e: dbd = dbdreader.DBD('dbdreader/data/02450137.tcd', cacheDir='dbdreader/data/cac') assert e.value.value == dbdreader.DBD_ERROR_DECOMPRESSION_ERROR - - - + + + From 1fa35f69aa5c612f1c921beb5202c07c9b1c5d56 Mon Sep 17 00:00:00 2001 From: Jody Klymak Date: Fri, 3 Jul 2026 17:04:48 -0700 Subject: [PATCH 4/8] FIX: windows glob --- dbdreader/dbdreader.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/dbdreader/dbdreader.py b/dbdreader/dbdreader.py index 4764177..29ce71a 100644 --- a/dbdreader/dbdreader.py +++ b/dbdreader/dbdreader.py @@ -5,6 +5,7 @@ import time import numpy import glob +import fnmatch import sys import re import datetime @@ -407,6 +408,33 @@ def set_cachedir(cls, path, force_makedirs=False): +def _glob(pattern): + ''' Case-sensitive glob. + + glob.glob() matches filenames case-insensitively on platforms with a + case-insensitive filesystem (notably Windows). For glider data files + this is a problem: lower and upper case extensions (.sbd/.SBD, + .tbd/.TBD, and so on) denote different (full resolution vs. + compact/telemetered) data files and must not be confused. This + wrapper filters glob.glob()'s result so that only names matching the + pattern case-sensitively are kept, giving the same result on every + platform. + + Parameters + ---------- + pattern : str + search pattern, as passed to glob.glob() + + Returns + ------- + list of str + filenames matching pattern, case-sensitively. + ''' + normalised_pattern = os.path.normpath(pattern) + return [fn for fn in glob.glob(pattern) + if fnmatch.fnmatchcase(os.path.normpath(fn), normalised_pattern)] + + class DBDList(list): ''' List that properly sorts dbd files. @@ -621,7 +649,7 @@ def get_filenames(self, pattern, filenames, cacheDir=None): if not pattern and not filenames: raise ValueError("Expected some pattern to search files for or file list.") if pattern: - all_filenames=DBDList(glob.glob(pattern)) + all_filenames=DBDList(_glob(pattern)) elif filenames: all_filenames=DBDList(filenames) else: @@ -1403,7 +1431,7 @@ def __init__(self,filenames=None,pattern=None,cacheDir=None,complemented_files_o if filenames: fns+=filenames if pattern: - fns+=glob.glob(pattern) + fns+=_glob(pattern) if len(fns)==0: raise DbdError(DBD_ERROR_NO_FILES_FOUND) fns.sort() From 872bbbc3158201023b6e53fc2d2b3601e3f1aa11 Mon Sep 17 00:00:00 2001 From: Jody Klymak Date: Fri, 3 Jul 2026 17:13:35 -0700 Subject: [PATCH 5/8] FIX: windows fix separator --- dbdreader/dbdreader.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/dbdreader/dbdreader.py b/dbdreader/dbdreader.py index 29ce71a..e4847f2 100644 --- a/dbdreader/dbdreader.py +++ b/dbdreader/dbdreader.py @@ -431,8 +431,13 @@ def _glob(pattern): filenames matching pattern, case-sensitively. ''' normalised_pattern = os.path.normpath(pattern) - return [fn for fn in glob.glob(pattern) - if fnmatch.fnmatchcase(os.path.normpath(fn), normalised_pattern)] + matches = [fn for fn in glob.glob(pattern) + if fnmatch.fnmatchcase(os.path.normpath(fn), normalised_pattern)] + # glob.glob() returns filenames built with the OS-native separator + # (e.g. backslashes on Windows), even when the pattern itself uses + # forward slashes. Normalise to forward slashes so that filenames + # are stable and comparable across platforms. + return [fn.replace(os.sep, '/') for fn in matches] class DBDList(list): From a774d6e4c6b220e22f2a67ac6ab175d7e084a7e6 Mon Sep 17 00:00:00 2001 From: Lucas Merckelbach Date: Mon, 6 Jul 2026 14:18:02 +0200 Subject: [PATCH 6/8] Include python 3.10 again --- dbdreader/dbdreader.py | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbdreader/dbdreader.py b/dbdreader/dbdreader.py index e4847f2..2856cfb 100644 --- a/dbdreader/dbdreader.py +++ b/dbdreader/dbdreader.py @@ -2061,7 +2061,7 @@ def _refresh_cache(self): time_limits[1]=min(time_limits[1],time_limits_dataset[1]) def _format_time(self,t,fmt): - tmp = datetime.datetime.fromtimestamp(t, datetime.UTC) + tmp = datetime.datetime.fromtimestamp(t, datetime.timezone.utc) return tmp.strftime(fmt) def _get_time_range(self,time_limits,fmt): diff --git a/pyproject.toml b/pyproject.toml index c7e4223..054c973 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ name = "dbdreader" version = "0.6.1" description = "A python module to access binary data files generated by Teledyne WebbResearch gliders" readme = "README.md" -requires-python = ">=3.11" +requires-python = ">=3.10" authors = [ {name = "Lucas Merckelbach", email = "lucas.merckelbach@hereon.de"} ] From aeac6b426048baec7d6c2249eff87c613e932cf2 Mon Sep 17 00:00:00 2001 From: Lucas Merckelbach Date: Mon, 6 Jul 2026 21:58:43 +0200 Subject: [PATCH 7/8] Adds python 3.14/3.10 and macos to python wheels. --- .github/workflows/pypi-publish.yml | 4 ++-- .github/workflows/test-publish.yml | 4 ++-- .github/workflows/tests.yml | 2 +- pyproject.toml | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml index 712c05d..5862c0b 100644 --- a/.github/workflows/pypi-publish.yml +++ b/.github/workflows/pypi-publish.yml @@ -12,7 +12,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, windows-latest, macos-latest] steps: - uses: actions/checkout@v4 @@ -20,7 +20,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 env: - CIBW_BUILD: "cp311-* cp312-* cp313-*" + CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-win32 *_i686 *musllinux*" - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/test-publish.yml b/.github/workflows/test-publish.yml index e893725..89c5327 100644 --- a/.github/workflows/test-publish.yml +++ b/.github/workflows/test-publish.yml @@ -9,7 +9,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, windows-latest] + os: [ubuntu-latest, macos-latest, windows-latest] steps: - uses: actions/checkout@v4 @@ -17,7 +17,7 @@ jobs: - name: Build wheels uses: pypa/cibuildwheel@v2.23.3 env: - CIBW_BUILD: "cp311-* cp312-* cp313-*" + CIBW_BUILD: "cp310-* cp311-* cp312-* cp313-* cp314-*" CIBW_SKIP: "*-win32 *_i686 *musllinux*" - uses: actions/upload-artifact@v4 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index cd8bc40..7116681 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -13,7 +13,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-latest, windows-latest] - python-version: ["3.11", "3.12", "3.13"] + python-version: ["3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 diff --git a/pyproject.toml b/pyproject.toml index 054c973..6e82832 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -72,6 +72,6 @@ exclude = ''' ''' [tool.mypy] -python_version = "3.13" +python_version = "3.14" strict = true # Enable all checks disallow_untyped_calls = true From 1366fe3e412cf6180d843ae84eca83cdae3baf5f Mon Sep 17 00:00:00 2001 From: Lucas Merckelbach Date: Mon, 6 Jul 2026 23:25:15 +0200 Subject: [PATCH 8/8] adds workflow_dispatch to tests.yml --- .github/workflows/tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 7116681..c313d3f 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -4,6 +4,7 @@ on: push: branches: [master] pull_request: + workflow_dispatch: jobs: test: