From 6be8831c31edfb3afcdddb82004d1527630f2245 Mon Sep 17 00:00:00 2001 From: Dezhen Lu Date: Wed, 11 Nov 2026 16:42:20 +0100 Subject: [PATCH 1/3] Keep scaled 32-bit xDR images exact in float64 GpuImageReader, batch_to_device and batch_to_device_stream promote a scaled integer image (BSCALE != 1 or BZERO != 0) to float32 whenever the on-disk item size is at most 4 bytes. float32 has a 24-bit significand, so BITPIX = 32 values above 2**24 are rounded. With the FITS unsigned 32-bit convention (BZERO = 2**31), 4294967295 is returned as 4294967296.0, which is outside the uint32 range. Astropy, CFITSIO and cuphoton.core.fits_io._logical_dtype all use float64 for scaled BITPIX = 32. Use float32 only for 8- and 16-bit integers, and add BITPIX = 32 cases to the promotion tests plus a GPU test that compares GpuImageReader with Astropy. Signed-off-by: Dezhen Lu --- src/cuphoton/xdr/prefetch.py | 5 +++-- src/cuphoton/xdr/reader.py | 3 ++- tests/xdr/test_prefetch.py | 28 ++++++++++++++++-------- tests/xdr/test_reader.py | 42 ++++++++++++++++++++++++++++++++++++ 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/src/cuphoton/xdr/prefetch.py b/src/cuphoton/xdr/prefetch.py index 539c3de7..600a20c2 100644 --- a/src/cuphoton/xdr/prefetch.py +++ b/src/cuphoton/xdr/prefetch.py @@ -165,7 +165,8 @@ def _probe_output_specs( dtype = np.dtype(dtype_char) scaled = hdu.image_bzero != 0.0 or hdu.image_bscale != 1.0 if scaled and dtype.kind in ("i", "u"): - dtype_char = "f4" if hdu.image_itemsize <= 4 else "f8" + # Float32 holds 8- and 16-bit integers exactly, as in Astropy. + dtype_char = "f4" if hdu.image_itemsize <= 2 else "f8" specs.append( _OutputSpec( shape=tuple(int(v) for v in hdu.image_shape), @@ -827,7 +828,7 @@ def _consume_image( if out.dtype.kind in ("i", "u"): out = out.astype( cp.float32 - if plan_item.image_itemsize <= 4 + if plan_item.image_itemsize <= 2 else cp.float64 ) if keepalive is not None: diff --git a/src/cuphoton/xdr/reader.py b/src/cuphoton/xdr/reader.py index 1d4b849a..e1796e2d 100644 --- a/src/cuphoton/xdr/reader.py +++ b/src/cuphoton/xdr/reader.py @@ -140,9 +140,10 @@ def read(self, *, out=None, stream=None, loader=None): byteswap_inplace(out, self.itemsize) if self.bzero != 0.0 or self.bscale != 1.0: # BZERO/BSCALE may be integer-coded; promote for scaling. + # Like Astropy, keep float32 only for 8- and 16-bit data. if out.dtype.kind in ("i", "u"): out = out.astype( - cp.float32 if self.itemsize <= 4 else cp.float64 + cp.float32 if self.itemsize <= 2 else cp.float64 ) apply_bzero_bscale(out, self.bzero, self.bscale) diff --git a/tests/xdr/test_prefetch.py b/tests/xdr/test_prefetch.py index 6aed2e0a..c6ccbd90 100644 --- a/tests/xdr/test_prefetch.py +++ b/tests/xdr/test_prefetch.py @@ -293,7 +293,7 @@ def _run_python_batches( @pytest.mark.parametrize( ("dtype_char", "itemsize", "expected"), - [("i2", 2, "f4"), ("i8", 8, "f8")], + [("i2", 2, "f4"), ("i4", 4, "f8"), ("i8", 8, "f8")], ) def test_probe_output_specs_promotes_scaled_integer_images( monkeypatch, dtype_char, itemsize, expected @@ -786,7 +786,17 @@ def inflate(d_concat, rel_offsets, **kwargs): assert captured["header_sizes"] is None -def test_consume_image_stages_raw_bytes_before_scaling(monkeypatch): +@pytest.mark.parametrize( + ("dtype_char", "raw_value", "bzero", "expected"), + [ + ("i2", 0, 32768.0, np.array([32768.0], dtype="f4")), + # FITS unsigned 32-bit values need float64 to stay exact. + ("i4", 2**24 + 1, 2.0**31, np.array([2**31 + 2**24 + 1], "f8")), + ], +) +def test_consume_image_stages_raw_bytes_before_scaling( + monkeypatch, dtype_char, raw_value, bzero, expected +): class NullStream: def __enter__(self): return self @@ -821,19 +831,19 @@ def scale(array, bzero, bscale): plan=None, heap_rel_offsets=None, heap_span_start=0, - heap_span_len=2, + heap_span_len=np.dtype(dtype_char).itemsize, image_shape=(1,), - image_dtype_char="i2", - image_itemsize=2, - image_bzero=32768.0, + image_dtype_char=dtype_char, + image_itemsize=np.dtype(dtype_char).itemsize, + image_bzero=bzero, ) - raw = np.array([0], dtype=">i2").view(np.uint8) - target = np.empty(1, dtype=np.float32) + raw = np.array([raw_value], dtype=f">{dtype_char}").view(np.uint8) + target = np.empty(1, dtype=expected.dtype) result = prefetch._consume_image(image, raw, target, stream=None) assert result is target - np.testing.assert_array_equal(target, np.array([32768.0], dtype="f4")) + np.testing.assert_array_equal(target, expected) def test_native_plan_stages_image_when_output_dtype_is_promoted(): diff --git a/tests/xdr/test_reader.py b/tests/xdr/test_reader.py index 4e295b81..f6fe0cee 100644 --- a/tests/xdr/test_reader.py +++ b/tests/xdr/test_reader.py @@ -116,6 +116,48 @@ def wait(): assert calls == [] +@pytest.mark.parametrize( + ("data", "bzero"), + [ + # Astropy writes the FITS unsigned convention, BZERO = 2**31. + (np.array([[0, 2**24 + 1, 2**32 - 1]], dtype=np.uint32), None), + (np.array([[2**24 + 1, -3]], dtype=np.int32), 0.5), + ], +) +def test_image_reader_scales_int32_images_without_rounding( + tmp_path, data, bzero +): + cp = pytest.importorskip("cupy") + kvikio = pytest.importorskip("kvikio") + defaults = pytest.importorskip("kvikio.defaults") + try: + if cp.cuda.runtime.getDeviceCount() == 0: + pytest.skip("CUDA device unavailable") + except cp.cuda.runtime.CUDARuntimeError: + pytest.skip("CUDA runtime unavailable") + from astropy.io import fits + + from cuphoton.xdr import GpuImageReader + + hdu = fits.ImageHDU(data) + if bzero is not None: + hdu.header["BZERO"] = bzero + path = tmp_path / "image.fits" + fits.HDUList([fits.PrimaryHDU(), hdu]).writeto(path) + + with ( + defaults.set("compat_mode", kvikio.CompatMode.ON), + fits.open(path) as hdus, + ): + # Read on the GPU first: loading .data makes Astropy rewrite the + # in-memory header (BITPIX -64, no BZERO) for non-integer scaling. + actual = cp.asnumpy(GpuImageReader(hdus[1]).read()) + expected = np.asarray(hdus[1].data, dtype=np.float64) + + assert actual.dtype == np.float64 + np.testing.assert_array_equal(actual, expected) + + @pytest.mark.parametrize("decode_fails", [False, True]) @pytest.mark.parametrize("sync_fails", [False, True]) @pytest.mark.parametrize("owns_loader", [False, True]) From 604bd181af8dd172e3a8df811fd843e827cd121b Mon Sep 17 00:00:00 2001 From: Dezhen Lu Date: Thu, 12 Nov 2026 20:28:52 +0100 Subject: [PATCH 2/3] Note the float64 result for scaled 32-bit xDR images in the changelog Scaled BITPIX = 32 images from GpuImageReader, batch_to_device and batch_to_device_stream now come back as float64 instead of float32, which changes the result dtype and doubles their device memory. Signed-off-by: Dezhen Lu Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0de1337a..54b16824 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,14 @@ sidecar creation and radio shard merging. See the [xBeam guide](docs/xbeam/README.md) for installation and examples. +### xDR + +- `GpuImageReader.read()`, `batch_to_device` and `batch_to_device_stream` + return scaled 32-bit integer images (BITPIX = 32 with BSCALE or BZERO) + as float64 instead of float32, so values above 2**24, including the + unsigned 32-bit convention, are no longer rounded. These images now use + twice the device memory. Scaled 8- and 16-bit images remain float32. + ## 0.1.3 These changes are relative to 0.1.2. Version 0.1.3 is a source-only release; From 5e4b62b8a17fd6538685152278a7e65107d464aa Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Mon, 16 Nov 2026 14:36:17 -0800 Subject: [PATCH 3/3] Run scaled xDR precision checks in GPU CI Include the scaled-int32 reader regression in the existing GPU test selection so its compatibility-mode cases run and cannot silently skip. Signed-off-by: Trent Nelson --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e62da101..a399a9e7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -411,6 +411,7 @@ jobs: tests/xpois/test_native_workspace.py tests/xrep/test_native_backend.py tests/xdr/test_native_postprocess.py + tests/xdr/test_reader.py::test_image_reader_scales_int32_images_without_rounding - name: Require native tests to execute run: | uv run --no-sync python - <<'PY'