diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 6d4543fa..5eeb1d44 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -105,6 +105,23 @@ logging for timings. See [hardware eligibility](components/xdr.md#hardware-eligi for buffer and tile-size requirements. FITS receipts retain requested choices, including choices that did not apply when the selected reader was Astropy. +## cuFile input on tmpfs fails to register + +cuFile (GPUDirect Storage) needs files on a disk-backed filesystem. If the +input is on `tmpfs` or `ramfs`, such as `/tmp` on many systems, opening or +reading it can fail with `HANDLE_NOT_REGISTERED`, or opening it with +`O_DIRECT` can fail with `Invalid argument` (`ramfs`, and `tmpfs` before +Linux 6.6, do not support `O_DIRECT`). Check the filesystem of the temporary +directory and of the input file: + +```bash +findmnt -T "${TMPDIR:-/tmp}" -o TARGET,FSTYPE +findmnt -T -o TARGET,FSTYPE +``` + +If either reports `tmpfs` or `ramfs`, point `TMPDIR` (or the input path) at +local disk storage and rerun. + ## A command rejected the input Use the component inspection or validation command before the expensive step. diff --git a/src/cuphoton/xbeam/cufile_io.py b/src/cuphoton/xbeam/cufile_io.py index 1edd8c3a..4d646aab 100644 --- a/src/cuphoton/xbeam/cufile_io.py +++ b/src/cuphoton/xbeam/cufile_io.py @@ -7,6 +7,7 @@ from __future__ import annotations import os +import re from collections import OrderedDict from collections.abc import Callable, Iterator from contextlib import contextmanager @@ -22,6 +23,53 @@ class CuFileUnavailable(RuntimeError): """The requested cuFile data path cannot be initialized.""" +_RAM_FILESYSTEMS = frozenset({"tmpfs", "ramfs"}) +# The kernel escapes space, tab, newline and backslash in mountinfo as \ooo. +_MOUNTINFO_ESCAPE = re.compile(r"\\([0-7]{3})") + + +def _filesystem_type( + path: str | os.PathLike[str], + mountinfo: str | os.PathLike[str] = "/proc/self/mountinfo", +) -> str | None: + """Return the filesystem type mounted at path, or None if unknown.""" + try: + target = os.path.realpath(path) + best_mount = "" + best_type = None + with open(mountinfo, encoding="utf-8") as lines: + for line in lines: + head, separator, tail = line.partition(" - ") + # mountinfo separates fields with ASCII spaces; other + # Unicode whitespace can be part of a mount-point name. + fields = head.split(" ") + if not separator or len(fields) < 5 or not tail.split(): + continue + mount = _MOUNTINFO_ESCAPE.sub( + lambda match: chr(int(match[1], 8)), fields[4] + ) + inside = target == mount or target.startswith( + mount.rstrip("/") + "/" + ) + if inside and len(mount) >= len(best_mount): + best_mount, best_type = mount, tail.split()[0] + return best_type + except (OSError, UnicodeError): + return None + + +def _ram_filesystem_hint(path: str | os.PathLike[str]) -> str: + """Explain a cuFile failure when the file lives in RAM-backed storage.""" + kind = _filesystem_type(path) + if kind not in _RAM_FILESYSTEMS: + return "" + return ( + f"; {path} is on {kind}, which cuFile does not support. Use a " + "disk-backed filesystem, for example by pointing TMPDIR (or the " + "input path) at local disk storage" + ) + + _DRIVER_LOCK = Lock() _DRIVER_USERS = 0 @@ -109,6 +157,7 @@ def open(self, path: str | os.PathLike[str]) -> CuFileHandle: except OSError as error: raise CuFileUnavailable( f"cannot open {selected} for direct cuFile input: {error}" + f"{_ram_filesystem_hint(selected)}" ) from error try: descriptor = self._cufile.Descr() @@ -122,6 +171,7 @@ def open(self, path: str | os.PathLike[str]) -> CuFileHandle: os.close(fd) raise CuFileUnavailable( f"cuFile cannot register {selected}: {error}" + f"{_ram_filesystem_hint(selected)}" ) from error result = CuFileHandle(selected, fd, handle) self._files.append(result) @@ -332,12 +382,13 @@ def read( except Exception as error: raise OSError( f"cuFile read failed for {file.path} at byte " - f"{file_offset}: {error}" + f"{file_offset}: {error}{_ram_filesystem_hint(file.path)}" ) from error if received < 0: raise OSError( f"cuFile read failed for {file.path} at byte " f"{file_offset}: status {received}" + f"{_ram_filesystem_hint(file.path)}" ) return received diff --git a/tests/xbeam/test_cufile_io.py b/tests/xbeam/test_cufile_io.py index 51407976..aad9c2f8 100644 --- a/tests/xbeam/test_cufile_io.py +++ b/tests/xbeam/test_cufile_io.py @@ -2,6 +2,7 @@ # # SPDX-License-Identifier: Apache-2.0 +import errno import os from collections import OrderedDict from contextlib import ExitStack @@ -298,3 +299,132 @@ def test_registered_reads_support_device_offsets(tmp_path): with pytest.raises(RuntimeError, match="not open"): session.read(handle, 65536, 4096, 8192) assert len(calls) == 2 + + +def test_register_failure_on_tmpfs_names_filesystem( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + path.touch() + + def refuse(_): + raise Exception("HANDLE_NOT_REGISTERED") + + session._cufile.handle_register = refuse + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs" + ) + with pytest.raises(CuFileUnavailable, match="tmpfs.*TMPDIR"): + session.open(path) + + +def test_open_failure_on_tmpfs_names_filesystem( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + path.touch() + + def refuse(*args): + raise OSError(errno.EINVAL, "Invalid argument") + + monkeypatch.setattr("cuphoton.xbeam.cufile_io.os.open", refuse) + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs" + ) + with pytest.raises( + CuFileUnavailable, match="Invalid argument.*tmpfs.*TMPDIR" + ): + session.open(path) + + +def test_read_failure_on_tmpfs_names_filesystem( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + handle = CuFileHandle(path, 3, 17) + + def fail(*args): + raise OSError("cuFile error") + + session._cufile.read = fail + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs" + ) + with pytest.raises(OSError, match="tmpfs.*TMPDIR"): + session.read(handle, 0, 16, 0) + + +def test_failure_on_disk_filesystem_keeps_original_message( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + path.touch() + + def refuse(_): + raise Exception("HANDLE_NOT_REGISTERED") + + session._cufile.handle_register = refuse + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "ext4" + ) + with pytest.raises(CuFileUnavailable) as raised: + session.open(path) + assert "tmpfs" not in str(raised.value) + + +def test_filesystem_type_uses_longest_mount_prefix(tmp_path): + from cuphoton.xbeam.cufile_io import _filesystem_type + + mountinfo = tmp_path / "mountinfo" + mountinfo.write_text( + "1 0 8:1 / / rw - ext4 /dev/sda1 rw\n" + f"2 1 0:30 / {tmp_path} rw - tmpfs tmpfs rw\n" + f"3 2 0:31 / {tmp_path}/disk rw - xfs /dev/sdb rw\n" + ) + (tmp_path / "disk").mkdir() + assert _filesystem_type(tmp_path / "a", mountinfo) == "tmpfs" + assert _filesystem_type(tmp_path / "disk" / "a", mountinfo) == "xfs" + assert _filesystem_type(tmp_path / "a", tmp_path / "missing") is None + + +def test_filesystem_type_prefers_later_entry_for_same_mount(tmp_path): + from cuphoton.xbeam.cufile_io import _filesystem_type + + mountinfo = tmp_path / "mountinfo" + mountinfo.write_text( + "1 0 8:1 / / rw - ext4 /dev/sda1 rw\n" + f"2 1 0:30 / {tmp_path} rw - xfs /dev/sdb rw\n" + f"3 1 0:31 / {tmp_path} rw - tmpfs tmpfs rw\n" + ) + assert _filesystem_type(tmp_path / "a", mountinfo) == "tmpfs" + + +@pytest.mark.parametrize("mount_name", ["données raw", "données\u00a0raw"]) +def test_filesystem_type_decodes_escaped_mount_points(tmp_path, mount_name): + from cuphoton.xbeam.cufile_io import _filesystem_type + + mountinfo = tmp_path / "mountinfo" + escaped_name = mount_name.replace(" ", r"\040") + mountinfo.write_text( + "1 0 8:1 / / rw - ext4 /dev/sda1 rw\n" + f"2 1 0:30 / /mnt/{escaped_name} rw - tmpfs tmpfs rw\n" + ) + assert _filesystem_type(f"/mnt/{mount_name}/a", mountinfo) == "tmpfs" + + +def test_negative_read_status_on_tmpfs_names_filesystem( + fake_session, tmp_path, monkeypatch +): + session, _ = fake_session + path = tmp_path / "source.vdif" + handle = CuFileHandle(path, 3, 17) + session._cufile.read = lambda *args: -5 + monkeypatch.setattr( + "cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs" + ) + with pytest.raises(OSError, match="status -5.*tmpfs.*TMPDIR"): + session.read(handle, 0, 16, 0)