Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -411,6 +411,7 @@ jobs:
tests/xpois/test_native_workspace.py
tests/xrep/test_native_backend.py
tests/xdr/test_native_postprocess.py
tests/xdr/test_reader.py::test_image_reader_scales_int32_images_without_rounding
- name: Require native tests to execute
run: |
uv run --no-sync python - <<'PY'
Expand Down
8 changes: 8 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,14 @@
sidecar creation and radio shard merging. See the
[xBeam guide](docs/xbeam/README.md) for installation and examples.

### xDR

- `GpuImageReader.read()`, `batch_to_device` and `batch_to_device_stream`
return scaled 32-bit integer images (BITPIX = 32 with BSCALE or BZERO)
as float64 instead of float32, so values above 2**24, including the
unsigned 32-bit convention, are no longer rounded. These images now use
twice the device memory. Scaled 8- and 16-bit images remain float32.

## 0.1.3

These changes are relative to 0.1.2. Version 0.1.3 is a source-only release;
Expand Down
5 changes: 3 additions & 2 deletions src/cuphoton/xdr/prefetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,8 @@ def _probe_output_specs(
dtype = np.dtype(dtype_char)
scaled = hdu.image_bzero != 0.0 or hdu.image_bscale != 1.0
if scaled and dtype.kind in ("i", "u"):
dtype_char = "f4" if hdu.image_itemsize <= 4 else "f8"
# Float32 holds 8- and 16-bit integers exactly, as in Astropy.
dtype_char = "f4" if hdu.image_itemsize <= 2 else "f8"
specs.append(
_OutputSpec(
shape=tuple(int(v) for v in hdu.image_shape),
Expand Down Expand Up @@ -827,7 +828,7 @@ def _consume_image(
if out.dtype.kind in ("i", "u"):
out = out.astype(
cp.float32
if plan_item.image_itemsize <= 4
if plan_item.image_itemsize <= 2
else cp.float64
)
if keepalive is not None:
Expand Down
3 changes: 2 additions & 1 deletion src/cuphoton/xdr/reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -140,9 +140,10 @@ def read(self, *, out=None, stream=None, loader=None):
byteswap_inplace(out, self.itemsize)
if self.bzero != 0.0 or self.bscale != 1.0:
# BZERO/BSCALE may be integer-coded; promote for scaling.
# Like Astropy, keep float32 only for 8- and 16-bit data.
if out.dtype.kind in ("i", "u"):
out = out.astype(
cp.float32 if self.itemsize <= 4 else cp.float64
cp.float32 if self.itemsize <= 2 else cp.float64
)
apply_bzero_bscale(out, self.bzero, self.bscale)

Expand Down
28 changes: 19 additions & 9 deletions tests/xdr/test_prefetch.py
Original file line number Diff line number Diff line change
Expand Up @@ -293,7 +293,7 @@ def _run_python_batches(

@pytest.mark.parametrize(
("dtype_char", "itemsize", "expected"),
[("i2", 2, "f4"), ("i8", 8, "f8")],
[("i2", 2, "f4"), ("i4", 4, "f8"), ("i8", 8, "f8")],
)
def test_probe_output_specs_promotes_scaled_integer_images(
monkeypatch, dtype_char, itemsize, expected
Expand Down Expand Up @@ -786,7 +786,17 @@ def inflate(d_concat, rel_offsets, **kwargs):
assert captured["header_sizes"] is None


def test_consume_image_stages_raw_bytes_before_scaling(monkeypatch):
@pytest.mark.parametrize(
("dtype_char", "raw_value", "bzero", "expected"),
[
("i2", 0, 32768.0, np.array([32768.0], dtype="f4")),
# FITS unsigned 32-bit values need float64 to stay exact.
("i4", 2**24 + 1, 2.0**31, np.array([2**31 + 2**24 + 1], "f8")),
],
)
def test_consume_image_stages_raw_bytes_before_scaling(
monkeypatch, dtype_char, raw_value, bzero, expected
):
class NullStream:
def __enter__(self):
return self
Expand Down Expand Up @@ -821,19 +831,19 @@ def scale(array, bzero, bscale):
plan=None,
heap_rel_offsets=None,
heap_span_start=0,
heap_span_len=2,
heap_span_len=np.dtype(dtype_char).itemsize,
image_shape=(1,),
image_dtype_char="i2",
image_itemsize=2,
image_bzero=32768.0,
image_dtype_char=dtype_char,
image_itemsize=np.dtype(dtype_char).itemsize,
image_bzero=bzero,
)
raw = np.array([0], dtype=">i2").view(np.uint8)
target = np.empty(1, dtype=np.float32)
raw = np.array([raw_value], dtype=f">{dtype_char}").view(np.uint8)
target = np.empty(1, dtype=expected.dtype)

result = prefetch._consume_image(image, raw, target, stream=None)

assert result is target
np.testing.assert_array_equal(target, np.array([32768.0], dtype="f4"))
np.testing.assert_array_equal(target, expected)


def test_native_plan_stages_image_when_output_dtype_is_promoted():
Expand Down
42 changes: 42 additions & 0 deletions tests/xdr/test_reader.py
Original file line number Diff line number Diff line change
Expand Up @@ -116,6 +116,48 @@ def wait():
assert calls == []


@pytest.mark.parametrize(
("data", "bzero"),
[
# Astropy writes the FITS unsigned convention, BZERO = 2**31.
(np.array([[0, 2**24 + 1, 2**32 - 1]], dtype=np.uint32), None),
(np.array([[2**24 + 1, -3]], dtype=np.int32), 0.5),
],
)
def test_image_reader_scales_int32_images_without_rounding(
tmp_path, data, bzero
):
cp = pytest.importorskip("cupy")
kvikio = pytest.importorskip("kvikio")
defaults = pytest.importorskip("kvikio.defaults")
try:
if cp.cuda.runtime.getDeviceCount() == 0:
pytest.skip("CUDA device unavailable")
except cp.cuda.runtime.CUDARuntimeError:
pytest.skip("CUDA runtime unavailable")
from astropy.io import fits

from cuphoton.xdr import GpuImageReader

hdu = fits.ImageHDU(data)
if bzero is not None:
hdu.header["BZERO"] = bzero
path = tmp_path / "image.fits"
fits.HDUList([fits.PrimaryHDU(), hdu]).writeto(path)

with (
defaults.set("compat_mode", kvikio.CompatMode.ON),
fits.open(path) as hdus,
):
# Read on the GPU first: loading .data makes Astropy rewrite the
# in-memory header (BITPIX -64, no BZERO) for non-integer scaling.
actual = cp.asnumpy(GpuImageReader(hdus[1]).read())
expected = np.asarray(hdus[1].data, dtype=np.float64)

assert actual.dtype == np.float64
np.testing.assert_array_equal(actual, expected)


@pytest.mark.parametrize("decode_fails", [False, True])
@pytest.mark.parametrize("sync_fails", [False, True])
@pytest.mark.parametrize("owns_loader", [False, True])
Expand Down
Loading