Skip to content

Commit ee17bbe

Browse files
committed
fix(io): sync local files to disk before the catalog commit
PyArrowFileIO wrote local files without fsync, so a power loss after an acknowledged commit could leave the catalog pointing at an empty metadata.json. Flush each file and its directory entry to disk when the stream is closed.
1 parent e3fbd94 commit ee17bbe

2 files changed

Lines changed: 67 additions & 1 deletion

File tree

‎pyiceberg/io/pyarrow.py‎

Lines changed: 46 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,11 +248,56 @@ def __init__(self, field: pa.Field, *args: Any):
248248
super().__init__(*args)
249249

250250

251+
def _fsync_dir(path: str) -> None:
252+
"""Make the entries of a local directory durable."""
253+
if os.name == "nt":
254+
# A directory cannot be opened on Windows, and NTFS journals its directory entries
255+
return
256+
fd = os.open(path, os.O_RDONLY)
257+
try:
258+
os.fsync(fd)
259+
finally:
260+
os.close(fd)
261+
262+
263+
class _SyncOnCloseOutputStream(pyarrow.BufferedOutputStream):
264+
"""A buffered local output stream that is flushed to disk when it is closed.
265+
266+
A catalog may point at a file as soon as it has been written, so the contents and the directory entry
267+
must survive a power loss before the commit, as an acknowledged PUT does on an object store.
268+
"""
269+
270+
def __init__(self, path: str, buffer_size: int):
271+
self._raw = pyarrow.OSFile(path, mode="wb")
272+
super().__init__(self._raw, buffer_size)
273+
self._path = path
274+
275+
def close(self) -> None:
276+
if self.closed:
277+
return
278+
try:
279+
self.flush()
280+
os.fsync(self._raw.fileno())
281+
finally:
282+
super().close()
283+
_fsync_dir(os.path.dirname(self._path) or ".")
284+
285+
251286
class PyArrowLocalFileSystem(pyarrow.fs.LocalFileSystem):
252287
def open_output_stream(self, path: str, *args: Any, **kwargs: Any) -> pyarrow.NativeFile:
253288
# In LocalFileSystem, parent directories must be first created before opening an output stream
289+
parent = os.path.dirname(path)
290+
missing: list[str] = []
291+
while parent and not os.path.isdir(parent):
292+
missing.append(parent)
293+
parent = os.path.dirname(parent)
254294
self.create_dir(os.path.dirname(path), recursive=True)
255-
return super().open_output_stream(path, *args, **kwargs)
295+
for directory in reversed(missing):
296+
_fsync_dir(os.path.dirname(directory) or ".")
297+
if args or kwargs.keys() - {"buffer_size"}:
298+
# Compression or metadata was requested, which only pyarrow's own stream supports; it is not synced
299+
return super().open_output_stream(path, *args, **kwargs)
300+
return _SyncOnCloseOutputStream(path, kwargs.get("buffer_size") or ONE_MEGABYTE)
256301

257302

258303
class PyArrowFile(InputFile, OutputFile):

‎tests/io/test_pyarrow.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,27 @@ def test_pyarrow_local_fs_can_create_path_without_parent_dir() -> None:
160160
pytest.fail("Failed to write to file without parent directory")
161161

162162

163+
def test_pyarrow_local_fs_syncs_file_and_new_directories_on_close(tmp_path: Path) -> None:
164+
"""Test a closed local file and the directory entries created for it are flushed to disk"""
165+
file_path = f"{tmp_path}/foo/bar/baz.txt"
166+
synced: list[int] = []
167+
real_fsync = os.fsync
168+
169+
def record_fsync(fd: int) -> None:
170+
synced.append(os.fstat(fd).st_ino)
171+
real_fsync(fd)
172+
173+
with patch("os.fsync", side_effect=record_fsync):
174+
with PyArrowFileIO().new_output(file_path).create() as f:
175+
f.write(b"foo")
176+
177+
expected = [os.stat(file_path).st_ino]
178+
if os.name != "nt":
179+
expected += [os.stat(directory).st_ino for directory in (tmp_path, tmp_path / "foo", tmp_path / "foo" / "bar")]
180+
assert sorted(synced) == sorted(expected)
181+
assert Path(file_path).read_bytes() == b"foo"
182+
183+
163184
def test_pyarrow_input_file() -> None:
164185
"""Test reading a file using PyArrowFile"""
165186

0 commit comments

Comments
 (0)