From c2be4360cfcfd07af4bd0b5d8979afa994c394de Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Fri, 13 Nov 2026 20:11:17 -0400 Subject: [PATCH 1/4] fix(xbeam): explain cuFile failures on tmpfs inputs cuFile cannot register or read files on tmpfs/ramfs, but the error only carried the raw HANDLE_NOT_REGISTERED code. Name the filesystem and point to TMPDIR in the existing open()/read() error messages, and document the check in the troubleshooting guide. Part of #185 Signed-off-by: Alex Castillo --- docs/troubleshooting.md | 13 ++++++ src/cuphoton/xbeam/cufile_io.py | 44 ++++++++++++++++++++- tests/xbeam/test_cufile_io.py | 70 +++++++++++++++++++++++++++++++++ 3 files changed, 126 insertions(+), 1 deletion(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 6d4543fa..8dbac98a 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -105,6 +105,19 @@ logging for timings. See [hardware eligibility](components/xdr.md#hardware-eligi for buffer and tile-size requirements. FITS receipts retain requested choices, including choices that did not apply when the selected reader was Astropy. +## cuFile input on tmpfs fails to register + +cuFile (GPUDirect Storage) needs files on a disk-backed filesystem. If the +input is on `tmpfs` or `ramfs`, such as `/tmp` on many systems, opening or +reading it can fail with `HANDLE_NOT_REGISTERED`. Check the filesystem: + +```bash +findmnt -T "$TMPDIR" -o TARGET,FSTYPE +``` + +If it reports `tmpfs` or `ramfs`, point `TMPDIR` (or the input path) at local +disk storage and rerun. + ## A command rejected the input Use the component inspection or validation command before the expensive step. diff --git a/src/cuphoton/xbeam/cufile_io.py b/src/cuphoton/xbeam/cufile_io.py index 1edd8c3a..021a7440 100644 --- a/src/cuphoton/xbeam/cufile_io.py +++ b/src/cuphoton/xbeam/cufile_io.py @@ -22,6 +22,47 @@ class CuFileUnavailable(RuntimeError): """The requested cuFile data path cannot be initialized.""" +_RAM_FILESYSTEMS = frozenset({"tmpfs", "ramfs"}) + + +def _filesystem_type( + path: str | os.PathLike[str], + mountinfo: str | os.PathLike[str] = "/proc/self/mountinfo", +) -> str | None: + """Return the filesystem type mounted at path, or None if unknown.""" + try: + target = os.path.realpath(path) + best_mount = "" + best_type = None + with open(mountinfo, encoding="utf-8") as lines: + for line in lines: + head, separator, tail = line.partition(" - ") + fields = head.split() + if not separator or len(fields) < 5 or not tail.split(): + continue + mount = fields[4].encode().decode("unicode_escape") + inside = target == mount or target.startswith( + mount.rstrip("/") + "/" + ) + if inside and len(mount) >= len(best_mount): + best_mount, best_type = mount, tail.split()[0] + return best_type + except (OSError, UnicodeError): + return None + + +def _ram_filesystem_hint(path: str | os.PathLike[str]) -> str: + """Explain a cuFile failure when the file lives in RAM-backed storage.""" + kind = _filesystem_type(path) + if kind not in _RAM_FILESYSTEMS: + return "" + return ( + f"; {path} is on {kind}, which cuFile does not support. Use a " + "disk-backed filesystem, for example by pointing TMPDIR (or the " + "input path) at local disk storage" + ) + + _DRIVER_LOCK = Lock() _DRIVER_USERS = 0 @@ -122,6 +163,7 @@ def open(self, path: str | os.PathLike[str]) -> CuFileHandle: os.close(fd) raise CuFileUnavailable( f"cuFile cannot register {selected}: {error}" + f"{_ram_filesystem_hint(selected)}" ) from error result = CuFileHandle(selected, fd, handle) self._files.append(result) @@ -332,7 +374,7 @@ def read( except Exception as error: raise OSError( f"cuFile read failed for {file.path} at byte " - f"{file_offset}: {error}" + f"{file_offset}: {error}{_ram_filesystem_hint(file.path)}" ) from error if received < 0: raise OSError( diff --git a/tests/xbeam/test_cufile_io.py b/tests/xbeam/test_cufile_io.py index 51407976..7ec2136d 100644 --- a/tests/xbeam/test_cufile_io.py +++ b/tests/xbeam/test_cufile_io.py @@ -298,3 +298,73 @@ def test_registered_reads_support_device_offsets(tmp_path): with pytest.raises(RuntimeError, match="not open"): session.read(handle, 65536, 4096, 8192) assert len(calls) == 2 + + +def test_register_failure_on_tmpfs_names_filesystem( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + path.touch() + + def refuse(_): + raise Exception("HANDLE_NOT_REGISTERED") + + session._cufile.handle_register = refuse + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs" + ) + with pytest.raises(CuFileUnavailable, match="tmpfs.*TMPDIR"): + session.open(path) + + +def test_read_failure_on_tmpfs_names_filesystem( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + handle = CuFileHandle(path, 3, 17) + + def fail(*args): + raise OSError("cuFile error") + + session._cufile.read = fail + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs" + ) + with pytest.raises(OSError, match="tmpfs.*TMPDIR"): + session.read(handle, 0, 16, 0) + + +def test_failure_on_disk_filesystem_keeps_original_message( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + path.touch() + + def refuse(_): + raise Exception("HANDLE_NOT_REGISTERED") + + session._cufile.handle_register = refuse + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "ext4" + ) + with pytest.raises(CuFileUnavailable) as raised: + session.open(path) + assert "tmpfs" not in str(raised.value) + + +def test_filesystem_type_uses_longest_mount_prefix(tmp_path): + from cuphoton.xbeam.cufile_io import _filesystem_type + + mountinfo = tmp_path / "mountinfo" + mountinfo.write_text( + "1 0 8:1 / / rw - ext4 /dev/sda1 rw\n" + f"2 1 0:30 / {tmp_path} rw - tmpfs tmpfs rw\n" + f"3 2 0:31 / {tmp_path}/disk rw - xfs /dev/sdb rw\n" + ) + (tmp_path / "disk").mkdir() + assert _filesystem_type(tmp_path / "a", mountinfo) == "tmpfs" + assert _filesystem_type(tmp_path / "disk" / "a", mountinfo) == "xfs" + assert _filesystem_type(tmp_path / "a", tmp_path / "missing") is None From 98214a46e78f104176fdabb11bef35126e685a9e Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sat, 14 Nov 2026 03:27:43 -0400 Subject: [PATCH 2/4] fix(xbeam): decode mountinfo escapes and hint on negative cuFile status Decode only the kernel's \ooo escapes in mountinfo mount points so non-ASCII paths match, and add the tmpfs hint to the negative-status read error too. Signed-off-by: Alex Castillo --- src/cuphoton/xbeam/cufile_io.py | 8 ++++++- tests/xbeam/test_cufile_io.py | 37 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/src/cuphoton/xbeam/cufile_io.py b/src/cuphoton/xbeam/cufile_io.py index 021a7440..96b5cf21 100644 --- a/src/cuphoton/xbeam/cufile_io.py +++ b/src/cuphoton/xbeam/cufile_io.py @@ -7,6 +7,7 @@ from __future__ import annotations import os +import re from collections import OrderedDict from collections.abc import Callable, Iterator from contextlib import contextmanager @@ -23,6 +24,8 @@ class CuFileUnavailable(RuntimeError): _RAM_FILESYSTEMS = frozenset({"tmpfs", "ramfs"}) +# The kernel escapes space, tab, newline and backslash in mountinfo as \ooo. +_MOUNTINFO_ESCAPE = re.compile(r"\\([0-7]{3})") def _filesystem_type( @@ -40,7 +43,9 @@ def _filesystem_type( fields = head.split() if not separator or len(fields) < 5 or not tail.split(): continue - mount = fields[4].encode().decode("unicode_escape") + mount = _MOUNTINFO_ESCAPE.sub( + lambda match: chr(int(match[1], 8)), fields[4] + ) inside = target == mount or target.startswith( mount.rstrip("/") + "/" ) @@ -380,6 +385,7 @@ def read( raise OSError( f"cuFile read failed for {file.path} at byte " f"{file_offset}: status {received}" + f"{_ram_filesystem_hint(file.path)}" ) return received diff --git a/tests/xbeam/test_cufile_io.py b/tests/xbeam/test_cufile_io.py index 7ec2136d..40fa6d7a 100644 --- a/tests/xbeam/test_cufile_io.py +++ b/tests/xbeam/test_cufile_io.py @@ -368,3 +368,40 @@ def test_filesystem_type_uses_longest_mount_prefix(tmp_path): assert _filesystem_type(tmp_path / "a", mountinfo) == "tmpfs" assert _filesystem_type(tmp_path / "disk" / "a", mountinfo) == "xfs" assert _filesystem_type(tmp_path / "a", tmp_path / "missing") is None + + +def test_filesystem_type_prefers_later_entry_for_same_mount(tmp_path): + from cuphoton.xbeam.cufile_io import _filesystem_type + + mountinfo = tmp_path / "mountinfo" + mountinfo.write_text( + "1 0 8:1 / / rw - ext4 /dev/sda1 rw\n" + f"2 1 0:30 / {tmp_path} rw - xfs /dev/sdb rw\n" + f"3 1 0:31 / {tmp_path} rw - tmpfs tmpfs rw\n" + ) + assert _filesystem_type(tmp_path / "a", mountinfo) == "tmpfs" + + +def test_filesystem_type_decodes_escaped_mount_points(tmp_path): + from cuphoton.xbeam.cufile_io import _filesystem_type + + mountinfo = tmp_path / "mountinfo" + mountinfo.write_text( + "1 0 8:1 / / rw - ext4 /dev/sda1 rw\n" + "2 1 0:30 / /mnt/données\\040raw rw - tmpfs tmpfs rw\n" + ) + assert _filesystem_type("/mnt/données raw/a", mountinfo) == "tmpfs" + + +def test_negative_read_status_on_tmpfs_names_filesystem( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + handle = CuFileHandle(path, 3, 17) + session._cufile.read = lambda *args: -5 + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs" + ) + with pytest.raises(OSError, match="status -5.*tmpfs.*TMPDIR"): + session.read(handle, 0, 16, 0) From b1a9814c88b5b3ff337234b3814544748013c3a8 Mon Sep 17 00:00:00 2001 From: Alex Castillo Date: Sat, 14 Nov 2026 03:33:57 -0400 Subject: [PATCH 3/4] fix(xbeam): hint on RAM filesystem when O_DIRECT open fails; refine tmpfs docs Signed-off-by: Alex Castillo --- docs/troubleshooting.md | 12 ++++++++---- src/cuphoton/xbeam/cufile_io.py | 1 + tests/xbeam/test_cufile_io.py | 21 +++++++++++++++++++++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 8dbac98a..5eeb1d44 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -109,14 +109,18 @@ including choices that did not apply when the selected reader was Astropy. cuFile (GPUDirect Storage) needs files on a disk-backed filesystem. If the input is on `tmpfs` or `ramfs`, such as `/tmp` on many systems, opening or -reading it can fail with `HANDLE_NOT_REGISTERED`. Check the filesystem: +reading it can fail with `HANDLE_NOT_REGISTERED`, or opening it with +`O_DIRECT` can fail with `Invalid argument` (`ramfs`, and `tmpfs` before +Linux 6.6, do not support `O_DIRECT`). Check the filesystem of the temporary +directory and of the input file: ```bash -findmnt -T "$TMPDIR" -o TARGET,FSTYPE +findmnt -T "${TMPDIR:-/tmp}" -o TARGET,FSTYPE +findmnt -T -o TARGET,FSTYPE ``` -If it reports `tmpfs` or `ramfs`, point `TMPDIR` (or the input path) at local -disk storage and rerun. +If either reports `tmpfs` or `ramfs`, point `TMPDIR` (or the input path) at +local disk storage and rerun. ## A command rejected the input diff --git a/src/cuphoton/xbeam/cufile_io.py b/src/cuphoton/xbeam/cufile_io.py index 96b5cf21..6bbb59c9 100644 --- a/src/cuphoton/xbeam/cufile_io.py +++ b/src/cuphoton/xbeam/cufile_io.py @@ -155,6 +155,7 @@ def open(self, path: str | os.PathLike[str]) -> CuFileHandle: except OSError as error: raise CuFileUnavailable( f"cannot open {selected} for direct cuFile input: {error}" + f"{_ram_filesystem_hint(selected)}" ) from error try: descriptor = self._cufile.Descr() diff --git a/tests/xbeam/test_cufile_io.py b/tests/xbeam/test_cufile_io.py index 40fa6d7a..842e4c1f 100644 --- a/tests/xbeam/test_cufile_io.py +++ b/tests/xbeam/test_cufile_io.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 +import errno import os from collections import OrderedDict from contextlib import ExitStack @@ -318,6 +319,26 @@ def refuse(_): session.open(path) +def test_open_failure_on_tmpfs_names_filesystem( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + path.touch() + + def refuse(*args): + raise OSError(errno.EINVAL, "Invalid argument") + + monkeypatch.setattr("cuphoton.xbeam.cufile_io.os.open", refuse) + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs" + ) + with pytest.raises( + CuFileUnavailable, match="Invalid argument.*tmpfs.*TMPDIR" + ): + session.open(path) + + def test_read_failure_on_tmpfs_names_filesystem( fake_session, tmp_path, monkeypatch ): From 1704973d05dabbb02dfe7886d9d1c5b23149ae24 Mon Sep 17 00:00:00 2001 From: Trent Nelson Date: Mon, 16 Nov 2026 11:56:24 -0800 Subject: [PATCH 4/4] Preserve Unicode whitespace in cuFile mount paths Split mountinfo fields only at ASCII spaces so RAM filesystem hints remain available for paths containing non-ASCII whitespace. Signed-off-by: Trent Nelson --- src/cuphoton/xbeam/cufile_io.py | 4 +++- tests/xbeam/test_cufile_io.py | 8 +++++--- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/cuphoton/xbeam/cufile_io.py b/src/cuphoton/xbeam/cufile_io.py index 6bbb59c9..4d646aab 100644 --- a/src/cuphoton/xbeam/cufile_io.py +++ b/src/cuphoton/xbeam/cufile_io.py @@ -40,7 +40,9 @@ def _filesystem_type( with open(mountinfo, encoding="utf-8") as lines: for line in lines: head, separator, tail = line.partition(" - ") - fields = head.split() + # mountinfo separates fields with ASCII spaces; other + # Unicode whitespace can be part of a mount-point name. + fields = head.split(" ") if not separator or len(fields) < 5 or not tail.split(): continue mount = _MOUNTINFO_ESCAPE.sub( diff --git a/tests/xbeam/test_cufile_io.py b/tests/xbeam/test_cufile_io.py index 842e4c1f..aad9c2f8 100644 --- a/tests/xbeam/test_cufile_io.py +++ b/tests/xbeam/test_cufile_io.py @@ -403,15 +403,17 @@ def test_filesystem_type_prefers_later_entry_for_same_mount(tmp_path): assert _filesystem_type(tmp_path / "a", mountinfo) == "tmpfs" -def test_filesystem_type_decodes_escaped_mount_points(tmp_path): +@pytest.mark.parametrize("mount_name", ["données raw", "données\u00a0raw"]) +def test_filesystem_type_decodes_escaped_mount_points(tmp_path, mount_name): from cuphoton.xbeam.cufile_io import _filesystem_type mountinfo = tmp_path / "mountinfo" + escaped_name = mount_name.replace(" ", r"\040") mountinfo.write_text( "1 0 8:1 / / rw - ext4 /dev/sda1 rw\n" - "2 1 0:30 / /mnt/données\\040raw rw - tmpfs tmpfs rw\n" + f"2 1 0:30 / /mnt/{escaped_name} rw - tmpfs tmpfs rw\n" ) - assert _filesystem_type("/mnt/données raw/a", mountinfo) == "tmpfs" + assert _filesystem_type(f"/mnt/{mount_name}/a", mountinfo) == "tmpfs" def test_negative_read_status_on_tmpfs_names_filesystem(