Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions docs/troubleshooting.md
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,23 @@ logging for timings. See [hardware eligibility](components/xdr.md#hardware-eligi
for buffer and tile-size requirements. FITS receipts retain requested choices,
including choices that did not apply when the selected reader was Astropy.

## cuFile input on tmpfs fails to register

cuFile (GPUDirect Storage) needs files on a disk-backed filesystem. If the
input is on `tmpfs` or `ramfs`, such as `/tmp` on many systems, opening or
reading it can fail with `HANDLE_NOT_REGISTERED`, or opening it with
`O_DIRECT` can fail with `Invalid argument` (`ramfs`, and `tmpfs` before
Linux 6.6, do not support `O_DIRECT`). Check the filesystem of the temporary
directory and of the input file:

```bash
findmnt -T "${TMPDIR:-/tmp}" -o TARGET,FSTYPE
findmnt -T <input path> -o TARGET,FSTYPE
```

If either reports `tmpfs` or `ramfs`, point `TMPDIR` (or the input path) at
local disk storage and rerun.

## A command rejected the input

Use the component inspection or validation command before the expensive step.
Expand Down
53 changes: 52 additions & 1 deletion src/cuphoton/xbeam/cufile_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
from __future__ import annotations

import os
import re
from collections import OrderedDict
from collections.abc import Callable, Iterator
from contextlib import contextmanager
Expand All @@ -22,6 +23,53 @@ class CuFileUnavailable(RuntimeError):
"""The requested cuFile data path cannot be initialized."""


_RAM_FILESYSTEMS = frozenset({"tmpfs", "ramfs"})
# The kernel escapes space, tab, newline and backslash in mountinfo as \ooo.
_MOUNTINFO_ESCAPE = re.compile(r"\\([0-7]{3})")


def _filesystem_type(
path: str | os.PathLike[str],
mountinfo: str | os.PathLike[str] = "/proc/self/mountinfo",
) -> str | None:
"""Return the filesystem type mounted at path, or None if unknown."""
try:
target = os.path.realpath(path)
best_mount = ""
best_type = None
with open(mountinfo, encoding="utf-8") as lines:
for line in lines:
head, separator, tail = line.partition(" - ")
# mountinfo separates fields with ASCII spaces; other
# Unicode whitespace can be part of a mount-point name.
fields = head.split(" ")
if not separator or len(fields) < 5 or not tail.split():
continue
mount = _MOUNTINFO_ESCAPE.sub(
lambda match: chr(int(match[1], 8)), fields[4]
)
inside = target == mount or target.startswith(
mount.rstrip("/") + "/"
)
if inside and len(mount) >= len(best_mount):
best_mount, best_type = mount, tail.split()[0]
return best_type
except (OSError, UnicodeError):
return None


def _ram_filesystem_hint(path: str | os.PathLike[str]) -> str:
"""Explain a cuFile failure when the file lives in RAM-backed storage."""
kind = _filesystem_type(path)
if kind not in _RAM_FILESYSTEMS:
return ""
return (
f"; {path} is on {kind}, which cuFile does not support. Use a "
"disk-backed filesystem, for example by pointing TMPDIR (or the "
"input path) at local disk storage"
)


_DRIVER_LOCK = Lock()
_DRIVER_USERS = 0

Expand Down Expand Up @@ -109,6 +157,7 @@ def open(self, path: str | os.PathLike[str]) -> CuFileHandle:
except OSError as error:
raise CuFileUnavailable(
f"cannot open {selected} for direct cuFile input: {error}"
f"{_ram_filesystem_hint(selected)}"
) from error
try:
descriptor = self._cufile.Descr()
Expand All @@ -122,6 +171,7 @@ def open(self, path: str | os.PathLike[str]) -> CuFileHandle:
os.close(fd)
raise CuFileUnavailable(
f"cuFile cannot register {selected}: {error}"
f"{_ram_filesystem_hint(selected)}"
Comment thread
tpn marked this conversation as resolved.
Resolved
) from error
result = CuFileHandle(selected, fd, handle)
self._files.append(result)
Expand Down Expand Up @@ -332,12 +382,13 @@ def read(
except Exception as error:
raise OSError(
f"cuFile read failed for {file.path} at byte "
f"{file_offset}: {error}"
f"{file_offset}: {error}{_ram_filesystem_hint(file.path)}"
) from error
if received < 0:
raise OSError(
f"cuFile read failed for {file.path} at byte "
f"{file_offset}: status {received}"
f"{_ram_filesystem_hint(file.path)}"
)
return received

Expand Down
130 changes: 130 additions & 0 deletions tests/xbeam/test_cufile_io.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#
# SPDX-License-Identifier: Apache-2.0

import errno
import os
from collections import OrderedDict
from contextlib import ExitStack
Expand Down Expand Up @@ -298,3 +299,132 @@ def test_registered_reads_support_device_offsets(tmp_path):
with pytest.raises(RuntimeError, match="not open"):
session.read(handle, 65536, 4096, 8192)
assert len(calls) == 2


def test_register_failure_on_tmpfs_names_filesystem(
fake_session, tmp_path, monkeypatch
):
session, _ = fake_session
path = tmp_path / "source.vdif"
path.touch()

def refuse(_):
raise Exception("HANDLE_NOT_REGISTERED")

session._cufile.handle_register = refuse
monkeypatch.setattr(
"cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs"
)
with pytest.raises(CuFileUnavailable, match="tmpfs.*TMPDIR"):
session.open(path)


def test_open_failure_on_tmpfs_names_filesystem(
fake_session, tmp_path, monkeypatch
):
session, _ = fake_session
path = tmp_path / "source.vdif"
path.touch()

def refuse(*args):
raise OSError(errno.EINVAL, "Invalid argument")

monkeypatch.setattr("cuphoton.xbeam.cufile_io.os.open", refuse)
monkeypatch.setattr(
"cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs"
)
with pytest.raises(
CuFileUnavailable, match="Invalid argument.*tmpfs.*TMPDIR"
):
session.open(path)


def test_read_failure_on_tmpfs_names_filesystem(
fake_session, tmp_path, monkeypatch
):
session, _ = fake_session
path = tmp_path / "source.vdif"
handle = CuFileHandle(path, 3, 17)

def fail(*args):
raise OSError("cuFile error")

session._cufile.read = fail
monkeypatch.setattr(
"cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs"
)
with pytest.raises(OSError, match="tmpfs.*TMPDIR"):
session.read(handle, 0, 16, 0)


def test_failure_on_disk_filesystem_keeps_original_message(
fake_session, tmp_path, monkeypatch
):
session, _ = fake_session
path = tmp_path / "source.vdif"
path.touch()

def refuse(_):
raise Exception("HANDLE_NOT_REGISTERED")

session._cufile.handle_register = refuse
monkeypatch.setattr(
"cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "ext4"
)
with pytest.raises(CuFileUnavailable) as raised:
session.open(path)
assert "tmpfs" not in str(raised.value)


def test_filesystem_type_uses_longest_mount_prefix(tmp_path):
from cuphoton.xbeam.cufile_io import _filesystem_type

mountinfo = tmp_path / "mountinfo"
mountinfo.write_text(
"1 0 8:1 / / rw - ext4 /dev/sda1 rw\n"
f"2 1 0:30 / {tmp_path} rw - tmpfs tmpfs rw\n"
f"3 2 0:31 / {tmp_path}/disk rw - xfs /dev/sdb rw\n"
)
(tmp_path / "disk").mkdir()
assert _filesystem_type(tmp_path / "a", mountinfo) == "tmpfs"
assert _filesystem_type(tmp_path / "disk" / "a", mountinfo) == "xfs"
assert _filesystem_type(tmp_path / "a", tmp_path / "missing") is None


def test_filesystem_type_prefers_later_entry_for_same_mount(tmp_path):
from cuphoton.xbeam.cufile_io import _filesystem_type

mountinfo = tmp_path / "mountinfo"
mountinfo.write_text(
"1 0 8:1 / / rw - ext4 /dev/sda1 rw\n"
f"2 1 0:30 / {tmp_path} rw - xfs /dev/sdb rw\n"
f"3 1 0:31 / {tmp_path} rw - tmpfs tmpfs rw\n"
)
assert _filesystem_type(tmp_path / "a", mountinfo) == "tmpfs"


@pytest.mark.parametrize("mount_name", ["données raw", "données\u00a0raw"])
def test_filesystem_type_decodes_escaped_mount_points(tmp_path, mount_name):
from cuphoton.xbeam.cufile_io import _filesystem_type

mountinfo = tmp_path / "mountinfo"
escaped_name = mount_name.replace(" ", r"\040")
mountinfo.write_text(
"1 0 8:1 / / rw - ext4 /dev/sda1 rw\n"
f"2 1 0:30 / /mnt/{escaped_name} rw - tmpfs tmpfs rw\n"
)
assert _filesystem_type(f"/mnt/{mount_name}/a", mountinfo) == "tmpfs"


def test_negative_read_status_on_tmpfs_names_filesystem(
fake_session, tmp_path, monkeypatch
):
session, _ = fake_session
path = tmp_path / "source.vdif"
handle = CuFileHandle(path, 3, 17)
session._cufile.read = lambda *args: -5
monkeypatch.setattr(
"cuphoton.xbeam.cufile_io._filesystem_type", lambda _: "tmpfs"
)
with pytest.raises(OSError, match="status -5.*tmpfs.*TMPDIR"):
session.read(handle, 0, 16, 0)
Loading